Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

49.6 KB
Raw Blame
#!/usr/bin/env python3
"""
Sample datasets to create subsets with configurable size.
Supports both full dataset and sampled subsets with seed-based reproducibility.
"""

import argparse
import json
import os
import random
import shutil
import sys
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple, Callable

sys.path.append(str(Path(__file__).parent.parent))


def load_json_data(file_path: Path) -> Any:
    """Load JSON data from file."""
    with open(file_path, "r", encoding="utf-8") as f:
        return json.load(f)


def load_jsonl_data(file_path: Path) -> List[Dict]:
    """Load JSONL data from file."""
    data = []
    with open(file_path, "r", encoding="utf-8") as f:
        for line in f:
            line = line.strip()
            if line:
                data.append(json.loads(line))
    return data


def save_json_data(data: Any, file_path: Path) -> None:
    """Save JSON data to file."""
    file_path.parent.mkdir(parents=True, exist_ok=True)
    with open(file_path, "w", encoding="utf-8") as f:
        json.dump(data, f, indent=2, ensure_ascii=False)


def save_jsonl_data(data: List[Dict], file_path: Path) -> None:
    """Save JSONL data to file."""
    file_path.parent.mkdir(parents=True, exist_ok=True)
    with open(file_path, "w", encoding="utf-8") as f:
        for item in data:
            f.write(json.dumps(item, ensure_ascii=False) + "\n")


def calculate_category_targets(
    sample_size: int,
    categories: List[str],
    print_info: bool = True
) -> Tuple[Dict[str, int], bool]:
    """
    Calculate category targets for stratified sampling.
    
    Args:
        sample_size: Total number of samples to take
        categories: List of category names
        print_info: Whether to print information
    
    Returns:
        Tuple of (category_targets dict, should_fallback_to_random bool)
    """
    num_categories = len(categories)
    if num_categories == 0:
        return {}, True
    
    base_per_category = sample_size // num_categories
    remainder = sample_size % num_categories
    
    if base_per_category == 0:
        if print_info:
            print(f"Warning: Sample size {sample_size} is too small for {num_categories} categories")
            print("Falling back to random sampling")
        return {}, True
    
    category_targets = {}
    for i, cat in enumerate(categories):
        category_targets[cat] = base_per_category + (1 if i < remainder else 0)
    
    if remainder > 0 and print_info:
        print(f"Cannot split {sample_size} QAs evenly into {num_categories} categories")
        print(f"Distributing {remainder} extra QA(s) to first {remainder} category(ies)")
    
    if print_info:
        print("Category targets:")
        for cat in categories:
            print(f"  {cat}: {category_targets[cat]} QAs")
    
    return category_targets, False


def stratified_sample_with_reallocation(
    sample_size: int,
    category_qas: Dict[str, List[Any]],
    seed: int,
    print_info: bool = True
) -> List[Any]:
    """
    Perform stratified sampling with remainder reallocation.
    
    Args:
        sample_size: Total number of samples to take
        category_qas: Dictionary mapping category to list of QAs
        seed: Random seed
        print_info: Whether to print information
    
    Returns:
        List of sampled QAs
    """
    random.seed(seed)
    categories = sorted(category_qas.keys())
    
    category_targets, should_fallback = calculate_category_targets(sample_size, categories, print_info)
    
    if should_fallback:
        all_qas = []
        for qas in category_qas.values():
            all_qas.extend(qas)
        random.seed(seed)
        if len(all_qas) > sample_size:
            return random.sample(all_qas, sample_size)
        return all_qas
    
    sampled_items = []
    remaining_quota = sample_size
    
    category_actual = {}
    for cat in categories:
        if cat not in category_targets or category_targets[cat] <= 0:
            category_actual[cat] = 0
            continue
        
        cat_qas = category_qas[cat].copy()
        random.shuffle(cat_qas)
        sample_count = min(len(cat_qas), category_targets[cat])
        category_actual[cat] = sample_count
        remaining_quota -= sample_count
        sampled_items.extend(cat_qas[:sample_count])
    
    if remaining_quota > 0 and print_info:
        print(f"Reallocating remaining {remaining_quota} QA(s) to categories with available QAs")
    
    category_available = {}
    for cat in categories:
        if cat in category_qas:
            total_available = len(category_qas[cat])
            used = category_actual.get(cat, 0)
            category_available[cat] = total_available - used
    
    while remaining_quota > 0:
        allocated_this_round = 0
        for cat in categories:
            if remaining_quota <= 0:
                break
            if category_available.get(cat, 0) > 0:
                cat_qas = category_qas[cat].copy()
                random.shuffle(cat_qas)
                for qa in cat_qas:
                    if qa not in sampled_items:
                        sampled_items.append(qa)
                        category_actual[cat] += 1
                        category_available[cat] -= 1
                        remaining_quota -= 1
                        allocated_this_round += 1
                        break
        
        if allocated_this_round == 0:
            if print_info:
                print(f"Warning: No more QAs available to sample. Stopping with {remaining_quota} unallocated.")
            break
    
    if print_info:
        print("Actual category counts after reallocation:")
        for cat in categories:
            print(f"  {cat}: {category_actual.get(cat, 0)} QAs")
    
    return sampled_items


def random_sample_qas(
    sample_size: int,
    all_qas: List[Any],
    seed: int
) -> List[Any]:
    """
    Perform random sampling of QAs.
    
    Args:
        sample_size: Number of samples to take
        all_qas: List of all available QAs
        seed: Random seed
    
    Returns:
        List of sampled QAs
    """
    random.seed(seed)
    if len(all_qas) > sample_size:
        return random.sample(all_qas, sample_size)
    return all_qas


def sample_docs_stratified(
    sample_size: int,
    category_qas: Dict[str, List[Tuple[Any, Any]]],
    doc_category_qas: Dict[Any, Dict[str, List[Any]]],
    all_doc_ids: List[Any],
    seed: int,
    print_info: bool = True
) -> List[Any]:
    """
    Sample documents using stratified sampling by selecting complete documents.
    
    Args:
        sample_size: Target number of QAs
        category_qas: Dict mapping category to list of (doc_id, qa) tuples
        doc_category_qas: Dict mapping doc_id to dict of category to QAs
        all_doc_ids: List of all document IDs
        seed: Random seed
        print_info: Whether to print information
    
    Returns:
        List of selected document IDs
    """
    random.seed(seed)
    categories = sorted(category_qas.keys())
    
    category_targets, should_fallback = calculate_category_targets(sample_size, categories, print_info)
    
    if should_fallback:
        return sample_docs_random(sample_size, doc_category_qas, all_doc_ids, seed, print_info)
    
    selected_docs = []
    selected_qas_by_cat = {cat: 0 for cat in categories}
    doc_used = {doc_id: False for doc_id in all_doc_ids}
    
    for cat in categories:
        target = category_targets[cat]
        if target == 0:
            continue
        
        cat_qas = category_qas[cat].copy()
        random.shuffle(cat_qas)
        
        for doc_id, qa in cat_qas:
            if doc_used[doc_id]:
                continue
            
            doc_cat_qas = doc_category_qas[doc_id]
            new_count = selected_qas_by_cat[cat] + len(doc_cat_qas.get(cat, []))
            if new_count > target:
                continue
            
            selected_docs.append(doc_id)
            doc_used[doc_id] = True
            
            for c, qs in doc_cat_qas.items():
                selected_qas_by_cat[c] += len(qs)
            
            if selected_qas_by_cat[cat] >= target:
                break
    
    total_selected = sum(selected_qas_by_cat.values())
    if print_info:
        print(f"Sampled {len(selected_docs)} documents with {total_selected} QAs")
        for cat in categories:
            print(f"  {cat}: {selected_qas_by_cat[cat]} QAs (target: {category_targets[cat]})")
    
    return selected_docs


def sample_docs_random(
    sample_size: int,
    doc_qas_count: Dict[Any, int],
    all_doc_ids: List[Any],
    seed: int,
    print_info: bool = True
) -> List[Any]:
    """
    Sample documents using random sampling by selecting complete documents.
    
    Args:
        sample_size: Target number of QAs
        doc_qas_count: Dict mapping doc_id to number of valid QAs
        all_doc_ids: List of all document IDs
        seed: Random seed
        print_info: Whether to print information
    
    Returns:
        List of selected document IDs
    """
    random.seed(seed)
    shuffled_docs = all_doc_ids.copy()
    random.shuffle(shuffled_docs)
    
    selected_docs = []
    selected_qas_count = 0
    
    for doc_id in shuffled_docs:
        doc_qas = doc_qas_count.get(doc_id, 0)
        
        if doc_qas == 0:
            continue
        
        if selected_qas_count + doc_qas <= sample_size or not selected_docs:
            selected_docs.append(doc_id)
            selected_qas_count += doc_qas
        else:
            if selected_qas_count >= sample_size:
                break
    
    if print_info:
        print(f"Sampled {len(selected_docs)} documents with {selected_qas_count} QAs (seed={seed})")
    
    return selected_docs


def sample_locomo(
    input_dir: Path,
    output_dir: Path,
    sample_size: Optional[int] = None,
    num_docs: Optional[int] = None,
    seed: int = 42,
    sample_mode: str = "random"
) -> Dict[str, Any]:
    """Sample Locomo dataset with stratified sampling support."""
    input_file = input_dir / "locomo10.json"
    if not input_file.exists():
        raise FileNotFoundError(f"locomo10.json not found at {input_file}")
    
    data = load_json_data(input_file)
    if not isinstance(data, list):
        data = [data]
    
    original_num_docs = len(data)
    print(f"Locomo original size: {original_num_docs} documents")
    
    category_qas = {}
    doc_category_qas = []
    for doc in data:
        doc_cat_qas = {}
        if "qa" in doc:
            for q in doc["qa"]:
                cat = str(q.get("category"))
                if cat != "5":
                    if cat not in category_qas:
                        category_qas[cat] = []
                    category_qas[cat].append((doc, q))
                    if cat not in doc_cat_qas:
                        doc_cat_qas[cat] = []
                    doc_cat_qas[cat].append(q)
        doc_category_qas.append(doc_cat_qas)
    
    total_qas = sum(len(qas) for qas in category_qas.values())
    categories = sorted(category_qas.keys())
    print(f"Total QAs (excluding category 5): {total_qas}")
    print(f"Categories: {categories}")
    for cat in categories:
        print(f"  Category {cat}: {len(category_qas[cat])} QAs")
    
    is_full = (sample_size is None and num_docs is None)
    if is_full:
        selected_docs = data
        print("Using full Locomo dataset")
    else:
        if num_docs is not None:
            if num_docs >= original_num_docs:
                selected_docs = data
                print("Using all documents")
            else:
                random.seed(seed)
                selected_docs = random.sample(data, num_docs)
                print(f"Sampled {len(selected_docs)} documents (seed={seed})")
            
            if sample_size is not None:
                print(f"Further sampling {sample_size} QAs from selected documents (mode: {sample_mode})")
                
                selected_doc_category_qas = {}
                selected_doc_indices = [data.index(doc) for doc in selected_docs]
                
                for doc_idx in selected_doc_indices:
                    doc = data[doc_idx]
                    doc_cat_qas = doc_category_qas[doc_idx]
                    for cat, qs in doc_cat_qas.items():
                        if cat not in selected_doc_category_qas:
                            selected_doc_category_qas[cat] = []
                        for q in qs:
                            selected_doc_category_qas[cat].append((doc_idx, q))
                
                if sample_mode == "stratified":
                    sampled_q_tuples = stratified_sample_with_reallocation(
                        sample_size, selected_doc_category_qas, seed
                    )
                    
                    keep_q_indices = set()
                    for doc_idx, q in sampled_q_tuples:
                        doc = data[doc_idx]
                        for q_idx, qa_item in enumerate(doc.get("qa", [])):
                            if qa_item == q:
                                keep_q_indices.add((doc_idx, q_idx))
                                break
                    
                    for doc in selected_docs:
                        doc_idx = data.index(doc)
                        new_qas = []
                        for q_idx, q in enumerate(doc.get("qa", [])):
                            if (doc_idx, q_idx) in keep_q_indices or str(q.get("category")) == "5":
                                new_qas.append(q)
                        doc["qa"] = new_qas
                
                if sample_mode == "random":
                    all_valid_q_indices = []
                    for doc_idx_in_selected, doc in enumerate(selected_docs):
                        doc_idx = data.index(doc)
                        doc_cat_qas = doc_category_qas[doc_idx]
                        for cat, qs in doc_cat_qas.items():
                            for q in qs:
                                for q_idx, qa_item in enumerate(doc.get("qa", [])):
                                    if qa_item == q:
                                        all_valid_q_indices.append((doc_idx_in_selected, q_idx))
                                        break
                    
                    sampled_q_indices = random_sample_qas(sample_size, all_valid_q_indices, seed)
                    keep_q_indices = set(sampled_q_indices)
                    
                    for doc_idx_in_selected, doc in enumerate(selected_docs):
                        new_qas = []
                        for q_idx, q in enumerate(doc.get("qa", [])):
                            if (doc_idx_in_selected, q_idx) in keep_q_indices or str(q.get("category")) == "5":
                                new_qas.append(q)
                        doc["qa"] = new_qas
        else:
            doc_qas_count = {}
            for doc_idx, doc_cat_qas in enumerate(doc_category_qas):
                count = 0
                for qs in doc_cat_qas.values():
                    count += len(qs)
                doc_qas_count[doc_idx] = count
            
            if sample_mode == "stratified":
                print(f"Using stratified sampling (seed={seed})")
                category_qas_with_indices = {}
                for cat, qas in category_qas.items():
                    category_qas_with_indices[cat] = []
                    for doc, q in qas:
                        doc_idx = data.index(doc)
                        category_qas_with_indices[cat].append((doc_idx, q))
                
                doc_category_qas_dict = {i: d for i, d in enumerate(doc_category_qas)}
                all_doc_indices = list(range(len(data)))
                
                selected_doc_indices = sample_docs_stratified(
                    sample_size, category_qas_with_indices, doc_category_qas_dict, all_doc_indices, seed
                )
                selected_docs = [data[i] for i in selected_doc_indices]
            
            if sample_mode == "random":
                print(f"Using random sampling (seed={seed})")
                all_doc_indices = list(range(len(data)))
                selected_doc_indices = sample_docs_random(
                    sample_size, doc_qas_count, all_doc_indices, seed
                )
                selected_docs = [data[i] for i in selected_doc_indices]
    
    output_data = selected_docs
    output_file = output_dir / "locomo10.json"
    save_json_data(output_data, output_file)
    
    sampled_qas = 0
    for doc in selected_docs:
        if "qa" in doc:
            for q in doc["qa"]:
                if str(q.get("category")) != "5":
                    sampled_qas += 1
    
    metadata = {
        "dataset": "Locomo",
        "original_num_docs": original_num_docs,
        "original_total_qas": total_qas,
        "sampled_num_docs": len(selected_docs),
        "sampled_total_qas": sampled_qas,
        "sample_size": sample_size,
        "num_docs": num_docs,
        "seed": seed,
        "sample_mode": sample_mode,
        "is_full": is_full,
        "note": "Category 5 questions are excluded from QA count"
    }
    
    return metadata


def sample_syllabusqa(
    input_dir: Path,
    output_dir: Path,
    sample_size: Optional[int] = None,
    num_docs: Optional[int] = None,
    seed: int = 42,
    sample_mode: str = "random"
) -> Dict[str, Any]:
    """Sample SyllabusQA dataset with stratified sampling support."""
    from collections import defaultdict
    import csv
    
    dataset_split_dir = input_dir / "data" / "dataset_split"
    if not dataset_split_dir.exists():
        raise FileNotFoundError(f"data/dataset_split not found at {dataset_split_dir}")
    
    all_data = []
    csv_files = ["train.csv", "val.csv", "test.csv"]
    for csv_file in csv_files:
        file_path = dataset_split_dir / csv_file
        if file_path.exists():
            with open(file_path, "r", encoding="utf-8") as f:
                reader = csv.DictReader(f)
                file_data = list(reader)
                for item in file_data:
                    item["_source_file"] = csv_file
                all_data.extend(file_data)
    
    doc_groups = defaultdict(list)
    for item in all_data:
        syllabus_name = item.get("syllabus_name", "unknown")
        doc_groups[syllabus_name].append(item)
    
    category_qas = {}
    doc_category_qas = {}
    for doc_name, items in doc_groups.items():
        doc_cat_qas = {}
        for item in items:
            q_type = item.get("question_type")
            if q_type != "no answer":
                if q_type not in category_qas:
                    category_qas[q_type] = []
                category_qas[q_type].append((doc_name, item))
                if q_type not in doc_cat_qas:
                    doc_cat_qas[q_type] = []
                doc_cat_qas[q_type].append(item)
        doc_category_qas[doc_name] = doc_cat_qas
    
    total_valid_qas = sum(len(qas) for qas in category_qas.values())
    categories = sorted(category_qas.keys())
    
    doc_valid_qas = {}
    for doc_name, items in doc_groups.items():
        valid_count = 0
        for item in items:
            if item.get("question_type") != "no answer":
                valid_count += 1
        doc_valid_qas[doc_name] = valid_count
    
    all_doc_names = list(doc_groups.keys())
    original_num_docs = len(all_doc_names)
    original_total_qas = len(all_data)
    print(f"SyllabusQA original size: {original_num_docs} documents, {original_total_qas} QAs (from {len(csv_files)} files)")
    print(f"Total valid QAs (excluding 'no answer'): {total_valid_qas}")
    print(f"Categories: {categories}")
    for cat in categories:
        print(f"  {cat}: {len(category_qas[cat])} QAs")
    
    is_full = (sample_size is None and num_docs is None)
    if is_full:
        selected_docs = all_doc_names
        print("Using full SyllabusQA dataset")
    else:
        if num_docs is not None:
            if num_docs >= original_num_docs:
                selected_docs = all_doc_names
                print("Using all documents")
            else:
                random.seed(seed)
                selected_docs = random.sample(all_doc_names, num_docs)
                print(f"Sampled {len(selected_docs)} documents (seed={seed})")
            
            if sample_size is not None:
                print(f"Further sampling {sample_size} QAs from selected documents (mode: {sample_mode})")
                
                selected_doc_category_qas = {}
                for doc_name in selected_docs:
                    doc_cat_qas = doc_category_qas[doc_name]
                    for cat, items in doc_cat_qas.items():
                        if cat not in selected_doc_category_qas:
                            selected_doc_category_qas[cat] = []
                        for item in items:
                            selected_doc_category_qas[cat].append(item)
                
                if sample_mode == "stratified":
                    sampled_items = stratified_sample_with_reallocation(
                        sample_size, selected_doc_category_qas, seed
                    )
                    
                    for doc_name in selected_docs:
                        doc_items = doc_groups[doc_name]
                        for item in doc_items:
                            if item.get("question_type") == "no answer":
                                sampled_items.append(item)
                    
                    new_doc_groups = defaultdict(list)
                    for item in sampled_items:
                        doc_name = item.get("syllabus_name", "unknown")
                        new_doc_groups[doc_name].append(item)
                    doc_groups = new_doc_groups
                
                if sample_mode == "random":
                    all_valid_items = []
                    for doc_name in selected_docs:
                        items = doc_groups[doc_name]
                        for item in items:
                            if item.get("question_type") != "no answer":
                                all_valid_items.append(item)
                    
                    sampled_items = random_sample_qas(sample_size, all_valid_items, seed)
                    
                    for doc_name in selected_docs:
                        items = doc_groups[doc_name]
                        for item in items:
                            if item.get("question_type") == "no answer":
                                sampled_items.append(item)
                    
                    new_doc_groups = defaultdict(list)
                    for item in sampled_items:
                        doc_name = item.get("syllabus_name", "unknown")
                        new_doc_groups[doc_name].append(item)
                    doc_groups = new_doc_groups
        else:
            if sample_mode == "stratified":
                print(f"Using stratified sampling (seed={seed})")
                selected_docs = sample_docs_stratified(
                    sample_size, category_qas, doc_category_qas, all_doc_names, seed
                )
            
            if sample_mode == "random":
                print(f"Using random sampling (seed={seed})")
                selected_docs = sample_docs_random(
                    sample_size, doc_valid_qas, all_doc_names, seed
                )
    
    selected_data = []
    for doc_name in selected_docs:
        selected_data.extend(doc_groups[doc_name])
    
    output_dir.mkdir(parents=True, exist_ok=True)
    for csv_file in csv_files:
        file_data = [item for item in selected_data if item.get("_source_file") == csv_file]
        if file_data:
            for item in file_data:
                item.pop("_source_file", None)
            output_file = output_dir / csv_file
            with open(output_file, "w", encoding="utf-8", newline="") as f:
                if file_data:
                    writer = csv.DictWriter(f, fieldnames=file_data[0].keys())
                    writer.writeheader()
                    writer.writerows(file_data)
            print(f"Saved {len(file_data)} samples to {csv_file}")
    
    syllabi_src = input_dir / "syllabi"
    syllabi_dst = output_dir / "syllabi"
    if syllabi_src.exists():
        syllabi_dst.mkdir(parents=True, exist_ok=True)
        
        syllabus_names = set()
        for doc_name in selected_docs:
            items = doc_groups[doc_name]
            for item in items:
                syllabus_name = item.get("syllabus_name")
                if syllabus_name:
                    syllabus_names.add(syllabus_name)
        
        print(f"Copying syllabi for {len(syllabus_names)} unique syllabus files")
        
        for subdir in ["pdf", "text", "word"]:
            src_subdir = syllabi_src / "syllabi_redacted" / subdir
            dst_subdir = syllabi_dst / "syllabi_redacted" / subdir
            if src_subdir.exists():
                dst_subdir.mkdir(parents=True, exist_ok=True)
                
                for syllabus_name in syllabus_names:
                    for ext in [".pdf", ".txt", ".docx"]:
                        src_file = src_subdir / f"{syllabus_name}{ext}"
                        if src_file.exists():
                            shutil.copy2(src_file, dst_subdir / f"{syllabus_name}{ext}")
                            print(f"Copied {subdir}/{syllabus_name}{ext}")
                            break
    
    sampled_valid_qas = 0
    for doc_name in selected_docs:
        items = doc_groups[doc_name]
        for item in items:
            if item.get("question_type") != "no answer":
                sampled_valid_qas += 1
    
    metadata = {
        "dataset": "SyllabusQA",
        "original_num_docs": original_num_docs,
        "original_total_qas": original_total_qas,
        "original_valid_qas": total_valid_qas,
        "sampled_num_docs": len(selected_docs),
        "sampled_total_qas": len(selected_data),
        "sampled_valid_qas": sampled_valid_qas,
        "sample_size": sample_size,
        "num_docs": num_docs,
        "seed": seed,
        "sample_mode": sample_mode,
        "is_full": is_full,
        "note": "'no answer' type questions are excluded from QA count"
    }
    
    return metadata


def sample_qasper(
    input_dir: Path,
    output_dir: Path,
    sample_size: Optional[int] = None,
    num_docs: Optional[int] = None,
    seed: int = 42,
    sample_mode: str = "random"
) -> Dict[str, Any]:
    """Sample Qasper dataset with stratified sampling support."""
    json_files = ["qasper-train-v0.3.json", "qasper-dev-v0.3.json", "qasper-test-v0.3.json"]
    all_paper_ids = []
    paper_data_map = {}
    
    category_qas = {}
    paper_category_qas = {}
    
    for json_file in json_files:
        file_path = input_dir / json_file
        if file_path.exists():
            data = load_json_data(file_path)
            for paper_id, paper_data in data.items():
                all_paper_ids.append(paper_id)
                paper_data_map[paper_id] = (paper_data, json_file)
                
                paper_cat_qas = {}
                if "qas" in paper_data:
                    for qa_item in paper_data["qas"]:
                        is_unanswerable = all(
                            ans.get("answer", {}).get("unanswerable", False)
                            for ans in qa_item.get("answers", [])
                        )
                        if is_unanswerable:
                            continue
                        
                        answer_types = set()
                        for ans in qa_item.get("answers", []):
                            ans_obj = ans.get("answer", {})
                            if ans_obj.get("unanswerable", False):
                                continue
                            if ans_obj.get("extractive_spans"):
                                answer_types.add("extractive")
                            elif ans_obj.get("free_form_answer", "").strip():
                                answer_types.add("free_form")
                            elif ans_obj.get("yes_no") is not None:
                                answer_types.add("yes_no")
                        
                        primary_type = next(iter(answer_types), "extractive")
                        if primary_type not in category_qas:
                            category_qas[primary_type] = []
                        category_qas[primary_type].append((paper_id, qa_item))
                        
                        if primary_type not in paper_cat_qas:
                            paper_cat_qas[primary_type] = []
                        paper_cat_qas[primary_type].append(qa_item)
                
                paper_category_qas[paper_id] = paper_cat_qas
    
    original_num_docs = len(all_paper_ids)
    print(f"Qasper original size: {original_num_docs} documents (from {len(json_files)} files)")
    
    total_qas = sum(len(qas) for qas in category_qas.values())
    categories = sorted(category_qas.keys())
    print(f"Total QAs (excluding unanswerable): {total_qas}")
    print(f"Categories: {categories}")
    for cat in categories:
        print(f"  {cat}: {len(category_qas[cat])} QAs")
    
    is_full = (sample_size is None and num_docs is None)
    if is_full:
        selected_ids = all_paper_ids
        print("Using full Qasper dataset")
    else:
        if num_docs is not None:
            if num_docs >= original_num_docs:
                selected_ids = all_paper_ids
                print("Using all documents")
            else:
                if sample_size is not None:
                    paper_qas_count = {}
                    for paper_id in all_paper_ids:
                        count = 0
                        for cat_qas in paper_category_qas[paper_id].values():
                            count += len(cat_qas)
                        paper_qas_count[paper_id] = count
                    
                    random.seed(seed)
                    shuffled_papers = all_paper_ids.copy()
                    random.shuffle(shuffled_papers)
                    shuffled_papers.sort(key=lambda pid: paper_qas_count[pid], reverse=True)
                    
                    selected_ids = shuffled_papers[:num_docs]
                    print(f"Sampled {len(selected_ids)} documents with highest QA counts (seed={seed})")
                else:
                    random.seed(seed)
                    selected_ids = random.sample(all_paper_ids, num_docs)
                    print(f"Sampled {len(selected_ids)} documents (seed={seed})")
            
            if sample_size is not None:
                print(f"Further sampling {sample_size} QAs from selected documents (mode: {sample_mode})")
                
                qa_with_indices = []
                for paper_id in selected_ids:
                    paper_data, source_file = paper_data_map[paper_id]
                    for i, qa_item in enumerate(paper_data.get("qas", [])):
                        is_unanswerable = all(
                            ans.get("answer", {}).get("unanswerable", False)
                            for ans in qa_item.get("answers", [])
                        )
                        if not is_unanswerable:
                            answer_types = set()
                            for ans in qa_item.get("answers", []):
                                ans_obj = ans.get("answer", {})
                                if ans_obj.get("unanswerable", False):
                                    continue
                                if ans_obj.get("extractive_spans"):
                                    answer_types.add("extractive")
                                elif ans_obj.get("free_form_answer", "").strip():
                                    answer_types.add("free_form")
                                elif ans_obj.get("yes_no") is not None:
                                    answer_types.add("yes_no")
                            primary_type = next(iter(answer_types), "extractive")
                            qa_with_indices.append((paper_id, primary_type, i, qa_item))
                
                if sample_mode == "stratified":
                    selected_doc_category_qas = {}
                    for paper_id, cat, i, qa_item in qa_with_indices:
                        if cat not in selected_doc_category_qas:
                            selected_doc_category_qas[cat] = []
                        selected_doc_category_qas[cat].append((paper_id, i, qa_item))
                    
                    category_targets, should_fallback = calculate_category_targets(
                        sample_size, sorted(selected_doc_category_qas.keys())
                    )
                    
                    if not should_fallback:
                        random.seed(seed)
                        sampled_qas_indices = set()
                        remaining_quota = sample_size
                        
                        category_actual = {}
                        cats = sorted(selected_doc_category_qas.keys())
                        for cat in cats:
                            if cat not in category_targets or category_targets[cat] <= 0:
                                category_actual[cat] = 0
                                continue
                            
                            cat_qas = selected_doc_category_qas[cat].copy()
                            random.shuffle(cat_qas)
                            sample_count = min(len(cat_qas), category_targets[cat])
                            category_actual[cat] = sample_count
                            remaining_quota -= sample_count
                            
                            for paper_id, i, qa_item in cat_qas[:sample_count]:
                                sampled_qas_indices.add((paper_id, i))
                        
                        if remaining_quota > 0:
                            print(f"Reallocating remaining {remaining_quota} QA(s) to categories with available QAs")
                            
                            category_available = {}
                            for cat in cats:
                                if cat in selected_doc_category_qas:
                                    total_available = len(selected_doc_category_qas[cat])
                                    used = category_actual.get(cat, 0)
                                    category_available[cat] = total_available - used
                            
                            while remaining_quota > 0:
                                allocated_this_round = 0
                                for cat in cats:
                                    if remaining_quota <= 0:
                                        break
                                    if category_available.get(cat, 0) > 0:
                                        cat_qas = selected_doc_category_qas[cat].copy()
                                        random.shuffle(cat_qas)
                                        for paper_id, i, qa_item in cat_qas:
                                            if (paper_id, i) not in sampled_qas_indices:
                                                sampled_qas_indices.add((paper_id, i))
                                                category_actual[cat] += 1
                                                category_available[cat] -= 1
                                                remaining_quota -= 1
                                                allocated_this_round += 1
                                                break
                                
                                if allocated_this_round == 0:
                                    print(f"Warning: No more QAs available to sample. Stopping with {remaining_quota} unallocated.")
                                    break
                        
                        print("Actual category counts after reallocation:")
                        for cat in cats:
                            print(f"  {cat}: {category_actual.get(cat, 0)} QAs")
                    
                    for paper_id in selected_ids:
                        paper_data, source_file = paper_data_map[paper_id]
                        new_qas = []
                        
                        for i, qa_item in enumerate(paper_data.get("qas", [])):
                            is_unanswerable = all(
                                ans.get("answer", {}).get("unanswerable", False)
                                for ans in qa_item.get("answers", [])
                            )
                            if is_unanswerable or (paper_id, i) in sampled_qas_indices:
                                new_qas.append(qa_item)
                        
                        paper_data["qas"] = new_qas
                
                if sample_mode == "random":
                    sampled_qas = random_sample_qas(sample_size, qa_with_indices, seed)
                    
                    keep_qas_indices = set()
                    for paper_id, cat, i, qa_item in sampled_qas:
                        keep_qas_indices.add((paper_id, i))
                    
                    for paper_id in selected_ids:
                        paper_data, source_file = paper_data_map[paper_id]
                        new_qas = []
                        for i, qa_item in enumerate(paper_data.get("qas", [])):
                            is_unanswerable = all(
                                ans.get("answer", {}).get("unanswerable", False)
                                for ans in qa_item.get("answers", [])
                            )
                            if is_unanswerable or (paper_id, i) in keep_qas_indices:
                                new_qas.append(qa_item)
                        paper_data["qas"] = new_qas
        else:
            paper_qas_count = {}
            for paper_id in all_paper_ids:
                count = 0
                for cat_qas in paper_category_qas[paper_id].values():
                    count += len(cat_qas)
                paper_qas_count[paper_id] = count
            
            if sample_mode == "stratified":
                print(f"Using stratified sampling (seed={seed})")
                selected_ids = sample_docs_stratified(
                    sample_size, category_qas, paper_category_qas, all_paper_ids, seed
                )
            
            if sample_mode == "random":
                print(f"Using random sampling (seed={seed})")
                selected_ids = sample_docs_random(
                    sample_size, paper_qas_count, all_paper_ids, seed
                )
    
    output_dir.mkdir(parents=True, exist_ok=True)
    data_by_file = {}
    for paper_id in selected_ids:
        paper_data, source_file = paper_data_map[paper_id]
        if source_file not in data_by_file:
            data_by_file[source_file] = {}
        data_by_file[source_file][paper_id] = paper_data
    
    for json_file, output_data in data_by_file.items():
        output_file = output_dir / json_file
        save_json_data(output_data, output_file)
        print(f"Saved {len(output_data)} papers to {json_file}")
    
    sampled_qas = 0
    for paper_id in selected_ids:
        paper_data, source_file = paper_data_map[paper_id]
        if "qas" in paper_data:
            for qa_item in paper_data["qas"]:
                is_unanswerable = all(
                    ans.get("answer", {}).get("unanswerable", False)
                    for ans in qa_item.get("answers", [])
                )
                if not is_unanswerable:
                    sampled_qas += 1
    
    metadata = {
        "dataset": "Qasper",
        "original_num_docs": original_num_docs,
        "original_total_qas": total_qas,
        "sampled_num_docs": len(selected_ids),
        "sampled_total_qas": sampled_qas,
        "sample_size": sample_size,
        "num_docs": num_docs,
        "seed": seed,
        "sample_mode": sample_mode,
        "is_full": is_full,
        "note": "Unanswerable questions are excluded from QA count"
    }
    
    return metadata


def sample_financebench(
    input_dir: Path,
    output_dir: Path,
    sample_size: Optional[int] = None,
    num_docs: Optional[int] = None,
    seed: int = 42,
    sample_mode: str = "random"
) -> Dict[str, Any]:
    """Sample Financebench dataset with stratified sampling support."""
    from collections import defaultdict
    
    input_file = input_dir / "data" / "financebench_open_source.jsonl"
    if not input_file.exists():
        raise FileNotFoundError(f"financebench_open_source.jsonl not found at {input_file}")
    
    data = load_jsonl_data(input_file)
    
    doc_groups = defaultdict(list)
    for item in data:
        doc_name = item.get("doc_name", "unknown")
        doc_groups[doc_name].append(item)
    
    category_qas = {}
    doc_category_qas = {}
    for doc_name, items in doc_groups.items():
        doc_cat_qas = {}
        for item in items:
            q_type = item.get("question_type", "domain-relevant")
            if q_type not in category_qas:
                category_qas[q_type] = []
            category_qas[q_type].append((doc_name, item))
            if q_type not in doc_cat_qas:
                doc_cat_qas[q_type] = []
            doc_cat_qas[q_type].append(item)
        doc_category_qas[doc_name] = doc_cat_qas
    
    all_doc_names = list(doc_groups.keys())
    original_num_docs = len(all_doc_names)
    original_total_qas = len(data)
    total_qas = sum(len(qas) for qas in category_qas.values())
    categories = sorted(category_qas.keys())
    print(f"Financebench original size: {original_num_docs} documents, {original_total_qas} QAs")
    print(f"Categories: {categories}")
    for cat in categories:
        print(f"  {cat}: {len(category_qas[cat])} QAs")
    
    is_full = (sample_size is None and num_docs is None)
    if is_full:
        selected_docs = all_doc_names
        print("Using full Financebench dataset")
    else:
        if num_docs is not None:
            if num_docs >= original_num_docs:
                selected_docs = all_doc_names
                print("Using all documents")
            else:
                random.seed(seed)
                sorted_docs = sorted(all_doc_names, key=lambda x: len(doc_groups[x]), reverse=True)
                random.shuffle(sorted_docs)
                selected_docs = sorted(sorted_docs, key=lambda x: len(doc_groups[x]), reverse=True)[:num_docs]
                print(f"Sampled {len(selected_docs)} documents with highest QA counts (seed={seed})")
                print("Selected documents:")
                for doc in selected_docs:
                    print(f"  {doc}: {len(doc_groups[doc])} QAs")
            
            if sample_size is not None:
                print(f"Further sampling {sample_size} QAs from selected documents (mode: {sample_mode})")
                
                selected_doc_category_qas = {}
                for doc_name in selected_docs:
                    doc_cat_qas = doc_category_qas[doc_name]
                    for cat, items in doc_cat_qas.items():
                        if cat not in selected_doc_category_qas:
                            selected_doc_category_qas[cat] = []
                        for item in items:
                            selected_doc_category_qas[cat].append(item)
                
                if sample_mode == "stratified":
                    sampled_items = stratified_sample_with_reallocation(
                        sample_size, selected_doc_category_qas, seed
                    )
                    
                    new_doc_groups = defaultdict(list)
                    for item in sampled_items:
                        doc_name = item.get("doc_name", "unknown")
                        new_doc_groups[doc_name].append(item)
                    doc_groups = new_doc_groups
                
                if sample_mode == "random":
                    all_items = []
                    for doc_name in selected_docs:
                        items = doc_groups[doc_name]
                        all_items.extend(items)
                    
                    sampled_items = random_sample_qas(sample_size, all_items, seed)
                    
                    new_doc_groups = defaultdict(list)
                    for item in sampled_items:
                        doc_name = item.get("doc_name", "unknown")
                        new_doc_groups[doc_name].append(item)
                    doc_groups = new_doc_groups
        else:
            doc_qas_count = {doc_name: len(items) for doc_name, items in doc_groups.items()}
            
            if sample_mode == "stratified":
                print(f"Using stratified sampling (seed={seed})")
                selected_docs = sample_docs_stratified(
                    sample_size, category_qas, doc_category_qas, all_doc_names, seed
                )
            
            if sample_mode == "random":
                print(f"Using random sampling (seed={seed})")
                selected_docs = sample_docs_random(
                    sample_size, doc_qas_count, all_doc_names, seed
                )
    
    selected_data = []
    for doc_name in selected_docs:
        selected_data.extend(doc_groups[doc_name])
    
    output_file = output_dir / "financebench_open_source.jsonl"
    save_jsonl_data(selected_data, output_file)
    
    pdfs_src = input_dir / "pdfs"
    pdfs_dst = output_dir / "pdfs"
    
    if pdfs_src.exists():
        pdfs_dst.mkdir(parents=True, exist_ok=True)
        
        for doc_name in selected_docs:
            src_pdf = pdfs_src / f"{doc_name}.pdf"
            if src_pdf.exists():
                shutil.copy2(src_pdf, pdfs_dst / f"{doc_name}.pdf")
                print(f"Copied PDF: {doc_name}.pdf")
    
    metadata = {
        "dataset": "Financebench",
        "original_num_docs": original_num_docs,
        "original_total_qas": original_total_qas,
        "sampled_num_docs": len(selected_docs),
        "sampled_total_qas": len(selected_data),
        "sample_size": sample_size,
        "num_docs": num_docs,
        "seed": seed,
        "sample_mode": sample_mode,
        "is_full": is_full
    }
    
    return metadata


DATASET_SAMPLERS = {
    "Locomo": sample_locomo,
    "SyllabusQA": sample_syllabusqa,
    "Qasper": sample_qasper,
    "FinanceBench": sample_financebench,
}


def sample_dataset(
    dataset_name: str,
    input_dir: Path,
    output_dir: Path,
    sample_size: Optional[int] = None,
    num_docs: Optional[int] = None,
    seed: int = 42,
    sample_mode: str = "stratified"
) -> bool:
    """Sample a single dataset."""
    if dataset_name not in DATASET_SAMPLERS:
        print(f"Unknown dataset: {dataset_name}")
        return False
    
    print(f"\nProcessing {dataset_name}...")
    print(f"Input: {input_dir}")
    print(f"Output: {output_dir}")
    
    try:
        sampler = DATASET_SAMPLERS[dataset_name]
        metadata = sampler(input_dir, output_dir, sample_size, num_docs, seed, sample_mode)
        
        metadata_file = output_dir / "sampling_metadata.json"
        save_json_data(metadata, metadata_file)
        print(f"✓ Saved metadata to {metadata_file}")
        
        return True
    except Exception as e:
        print(f"Error sampling {dataset_name}: {e}")
        import traceback
        traceback.print_exc()
        return False


def main():
    parser = argparse.ArgumentParser(
        description="Sample datasets for RAG benchmark"
    )
    parser.add_argument(
        "--dataset", "-d",
        type=str,
        choices=list(DATASET_SAMPLERS.keys()) + ["all"],
        default="all",
        help="Dataset to sample (default: all)"
    )
    parser.add_argument(
        "--input-dir", "-i",
        type=Path,
        default=Path(__file__).parent.parent / "raw_data",
        help="Input directory with full datasets (default: raw_data/)"
    )
    parser.add_argument(
        "--output-dir", "-o",
        type=Path,
        default=Path(__file__).parent.parent / "datasets",
        help="Output directory for sampled datasets (default: datasets/)"
    )
    parser.add_argument(
        "--sample-size", "-n",
        type=int,
        default=None,
        help="Number of samples to use (default: use full dataset)"
    )
    parser.add_argument(
        "--num-docs",
        type=int,
        default=None,
        help="Number of documents to sample (for document-level sampling)"
    )
    parser.add_argument(
        "--seed", "-s",
        type=int,
        default=42,
        help="Random seed for reproducibility (default: 42)"
    )
    parser.add_argument(
        "--full",
        action="store_true",
        help="Use full dataset (overrides --sample-size)"
    )
    parser.add_argument(
        "--sample-mode",
        type=str,
        choices=["stratified", "random"],
        default="stratified",
        help="Sampling mode (default: stratified)"
    )
    
    args = parser.parse_args()
    
    if args.full:
        args.sample_size = None
        args.num_docs = None
    
    input_dir = args.input_dir.resolve()
    output_dir = args.output_dir.resolve()
    
    datasets = (
        list(DATASET_SAMPLERS.keys()) 
        if args.dataset == "all" 
        else [args.dataset]
    )
    
    print("=" * 60)
    print("RAG Benchmark Dataset Sampler")
    print("=" * 60)
    print(f"Input directory: {input_dir}")
    print(f"Output directory: {output_dir}")
    print(f"Sample size: {args.sample_size if args.sample_size else 'full'}")
    print(f"Number of docs: {args.num_docs if args.num_docs else 'not set'}")
    print(f"Sample mode: {args.sample_mode}")
    print(f"Random seed: {args.seed}")
    print(f"Datasets: {', '.join(datasets)}")
    print("=" * 60)
    
    success_count = 0
    for dataset in datasets:
        dataset_input_dir = input_dir / dataset
        dataset_output_dir = output_dir / dataset
        
        if sample_dataset(
            dataset,
            dataset_input_dir,
            dataset_output_dir,
            args.sample_size,
            args.num_docs,
            args.seed,
            args.sample_mode
        ):
            success_count += 1
    
    print("\n" + "=" * 60)
    print(f"Sampling complete: {success_count}/{len(datasets)} successful")
    print("=" * 60)
    
    return 0 if success_count == len(datasets) else 1


if __name__ == "__main__":
    sys.exit(main())