Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

15.2 KB
Raw Blame
# src/adapters/qasper_adapter.py
"""
Qasper Dataset Adapter

Qasper is an academic paper QA dataset containing 1585 NLP papers and 5049 questions.
Each question is answered by multiple annotators, with answer types including:
- extractive_spans: text spans extracted from the paper
- free_form_answer: free-form answers
- yes_no: yes/no answers
- unanswerable: no answer can be found in the paper

Dataset characteristics:
1. Each paper includes title, abstract, section content, and figure/table information
2. Each question may have answers from multiple annotators
3. Each answer has corresponding evidence (evidence text)

Adapter functions:
- data_prepare: Convert papers to Markdown format, preserving section structure
- load_and_transform: Parse QA data, preserving answer-evidence correspondence
- build_prompt: Build QA prompt
- post_process_answer: Post-process LLM output
"""

import json
import os
from typing import List, Dict, Any

from .base import BaseAdapter, StandardDoc, StandardSample, StandardQA

# Specific instructions for different answer types
CATEGORY_INSTRUCTIONS = {
    "extractive": """Extract the exact answer from the paper.
- Use EXACT wording from the context
- Do NOT rephrase or add explanation
- Provide concise, direct answer""",
    
    "free_form": """Answer using information from the paper.
- Use ONLY facts from the context
- You may rephrase or summarize in your own words
- Provide clear, complete answer
- Do NOT invent information""",
    
    "yes_no": """Answer Yes/No question based on the paper.
- First respond "Yes" or "No"
- Do NOT add explanation
- Use ONLY info from context
- Do NOT invent information"""
}

# Rule for when answer cannot be found
MISSING_RULE = "If no information is available to answer the question, write 'Not mentioned'."


class QasperAdapter(BaseAdapter):
    """
    Adapter specifically for processing Qasper dataset.
    
    Converts academic papers to Markdown documents with section structure,
    and converts QA data to standardized StandardSample format.
    
    Attributes:
        raw_file_path: Raw JSON data file path
        logger: Logger
    """
    
    def data_prepare(self, doc_dir: str) -> List[StandardDoc]:
        """
        Load raw data and convert to OpenViking-friendly format.
        
        Converts each paper to Markdown document, preserving:
        - Title (# Title)
        - Abstract (## Abstract)
        - Sections (## Section Name)
        - Figures and Tables (## Figures and Tables)
        
        Args:
            doc_dir: Document output directory path
            
        Returns:
            List[StandardDoc]: List of standardized document objects, each containing paper_id and document path
            
        Raises:
            FileNotFoundError: Raw data file not found
        """
        res: List[StandardDoc] = []
        data = {}

        if os.path.isdir(self.raw_file_path):
            json_files = [f for f in os.listdir(self.raw_file_path) if f.endswith('.json') and f != 'sampling_metadata.json']
            for json_file in json_files:
                file_path = os.path.join(self.raw_file_path, json_file)
                with open(file_path, 'r', encoding='utf-8') as f:
                    file_data = json.load(f)
                    data.update(file_data)
        else:
            if not os.path.exists(self.raw_file_path):
                raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}")
            with open(self.raw_file_path, 'r', encoding='utf-8') as f:
                data = json.load(f)

        os.makedirs(doc_dir, exist_ok=True)
        
        for paper_id, paper_data in data.items():
            doc_content = self._convert_paper_to_markdown(paper_id, paper_data)

            try:
                doc_path = os.path.join(doc_dir, f"{paper_id}_doc.md")
                with open(doc_path, "w", encoding="utf-8") as f:
                    f.write(doc_content)
                res.append(StandardDoc(paper_id, doc_path))
            except Exception as e:
                self.logger.error(f"[qasper adapter] doc:{paper_id} prepare error {e}")
                raise e
        return res

    def load_and_transform(self) -> List[StandardSample]:
        """
        Load raw JSON data and convert to standardized StandardSample object list.
        
        Processing logic:
        1. Iterate through QA list of each paper
        2. For each question, collect answers from all annotators
        3. Preserve answer-evidence correspondence (stored in metadata)
        4. Format question as "Based on the paper \"{title}\", {question}"
        
        Answer type handling:
        - extractive_spans: directly use extracted text spans
        - free_form_answer: use free-form answer
        - yes_no: convert to "Yes" or "No"
        - unanswerable: convert to "Not mentioned"
        
        Returns:
            List[StandardSample]: List of standardized sample objects
            
        Raises:
            FileNotFoundError: Raw data file not found
        """
        data = {}

        if os.path.isdir(self.raw_file_path):
            json_files = [f for f in os.listdir(self.raw_file_path) if f.endswith('.json') and f != 'sampling_metadata.json']
            for json_file in json_files:
                file_path = os.path.join(self.raw_file_path, json_file)
                with open(file_path, 'r', encoding='utf-8') as f:
                    file_data = json.load(f)
                    data.update(file_data)
        else:
            if not os.path.exists(self.raw_file_path):
                raise FileNotFoundError(f"Raw data file not found: {self.raw_file_path}")
            with open(self.raw_file_path, 'r', encoding='utf-8') as f:
                data = json.load(f)

        standard_samples = []

        for paper_id, paper_data in data.items():
            qa_pairs = []
            paper_title = paper_data.get("title", "Unknown Title")
            
            for qa_item in paper_data.get("qas", []):

                # --- Unanswerable filtering logic ---
                # Check if all answers are marked as unanswerable
                is_unanswerable = all(
                    ans.get("answer", {}).get("unanswerable", False) 
                    for ans in qa_item.get("answers", [])
                )
                if is_unanswerable:
                    continue
                # ------------------
                
                raw_question = qa_item.get("question", "")
                question_id = qa_item.get("question_id", "")
                # Append paper title to question for easier retrieval
                question = f'Based on the paper "{paper_title}", {raw_question}'
                
                gold_answers = []
                evidence_list = []
                answer_types = []
                answer_evidence_pairs = []
                
                # Iterate through all annotator answers
                for answer_wrapper in qa_item.get("answers", []):
                    answer_obj = answer_wrapper.get("answer", {})
                    
                    current_answer = None
                    answer_type = self._get_answer_type(answer_obj)
                    
                    # Process different answer types
                    if answer_obj.get("unanswerable", False):
                        current_answer = "Not mentioned"
                        gold_answers.append(current_answer)
                    else:
                        extractive_spans = answer_obj.get("extractive_spans", [])
                        free_form_answer = answer_obj.get("free_form_answer", "")
                        yes_no = answer_obj.get("yes_no")
                        
                        if extractive_spans:
                            valid_spans = [span.strip() for span in extractive_spans if span and span.strip()]
                            if valid_spans:
                                combined_answer = "; ".join(valid_spans)
                                gold_answers.append(combined_answer)
                                current_answer = combined_answer
                            else:
                                current_answer = None
                        elif free_form_answer and free_form_answer.strip():
                            current_answer = free_form_answer.strip()
                            gold_answers.append(current_answer)
                        elif yes_no is not None:
                            current_answer = "Yes" if yes_no else "No"
                            gold_answers.append(current_answer)
                    
                    # Collect evidence text
                    current_evidence = []
                    evidence = answer_obj.get("evidence", [])
                    for ev in evidence:
                        if ev and ev.strip():
                            current_evidence.append(ev)
                            if ev not in evidence_list:
                                evidence_list.append(ev)
                    
                    # Record answer type (deduplicated)
                    if answer_type not in answer_types:
                        answer_types.append(answer_type)
                    
                    # Save answer-evidence correspondence
                    if current_answer:
                        answer_evidence_pairs.append({
                            "answer": current_answer,
                            "evidence": current_evidence,
                            "answer_type": answer_type
                        })
                
                # If no answers, default to "Not mentioned"
                if not gold_answers:
                    gold_answers = ["Not mentioned"]
                
                # Deduplicate (preserve order)
                gold_answers = list(dict.fromkeys(gold_answers))
                
                qa_pairs.append(StandardQA(
                    question=question,
                    gold_answers=gold_answers,
                    evidence=evidence_list,
                    category=None,
                    metadata={
                        "question_id": question_id,
                        "answer_types": answer_types,
                        "answer_evidence_pairs": answer_evidence_pairs
                    }
                ))

            standard_samples.append(StandardSample(
                sample_id=paper_id,
                qa_pairs=qa_pairs
            ))

        return standard_samples
    
    def _get_answer_type(self, answer_obj: Dict[str, Any]) -> str:
        """
        Determine answer type from answer object.
        
        Args:
            answer_obj: Answer object, containing extractive_spans, free_form_answer, yes_no, etc.
            
        Returns:
            str: Answer type, possible values:
                - "unanswerable": cannot answer
                - "extractive": extractive answer
                - "free_form": free-form answer
                - "yes_no": yes/no answer
                - "unknown": unknown type
        """
        if answer_obj.get("unanswerable", False):
            return "unanswerable"
        if answer_obj.get("extractive_spans"):
            return "extractive"
        if answer_obj.get("free_form_answer", "").strip():
            return "free_form"
        if answer_obj.get("yes_no") is not None:
            return "yes_no"
        return "unknown"

    def _convert_paper_to_markdown(self, paper_id: str, paper_data: Dict[str, Any]) -> str:
        """
        Convert Qasper paper structure to Markdown string.
        
        Conversion format:
        ```markdown
        # {title}
        Paper ID: {paper_id}
        
        ## Abstract
        {abstract}
        
        ## Section Name 1
        {paragraph 1}
        {paragraph 2}
        
        ## Section Name 2
        ...
        
        ## Figures and Tables
        ### Figure 1
        Caption: {caption}
        File: {filename}
        
        ### Table 1
        Caption: {caption}
        File: {filename}
        ```
        
        Args:
            paper_id: Paper ID
            paper_data: Paper data, containing title, abstract, full_text, figures_and_tables
            
        Returns:
            str: Markdown formatted paper content
        """
        md_lines = []
        
        # Title
        title = paper_data.get("title", "Unknown Title")
        md_lines.append(f"# {title}")
        md_lines.append(f"Paper ID: {paper_id}\n")
        
        # Abstract
        abstract = paper_data.get("abstract", "")
        if abstract:
            md_lines.append("## Abstract")
            md_lines.append(abstract)
            md_lines.append("")
        
        # Main text sections
        full_text = paper_data.get("full_text", [])
        for section in full_text:
            section_name = section.get("section_name", "")
            paragraphs = section.get("paragraphs", [])
            
            if section_name:
                md_lines.append(f"## {section_name}")
            
            for para in paragraphs:
                if para and para.strip():
                    md_lines.append(para.strip())
                    md_lines.append("")
        
        # Figure and table information
        figures_and_tables = paper_data.get("figures_and_tables", [])
        if figures_and_tables:
            md_lines.append("## Figures and Tables")
            for idx, fig in enumerate(figures_and_tables, 1):
                caption = fig.get("caption", "")
                file_name = fig.get("file", "")
                
                # Determine if figure or table based on filename or caption
                if "Figure" in file_name or "figure" in caption.lower():
                    md_lines.append(f"### Figure {idx}")
                else:
                    md_lines.append(f"### Table {idx}")
                
                if caption:
                    md_lines.append(f"Caption: {caption}")
                if file_name:
                    md_lines.append(f"File: {file_name}")
                md_lines.append("")

        return "\n".join(md_lines)

    def build_prompt(self, qa: StandardQA, context_blocks: List[str]) -> tuple[str, Dict[str, Any]]:
        context_text = "\n\n".join(context_blocks) if context_blocks else "No relevant context found."
        
        answer_types = qa.metadata.get("answer_types", [])
        primary_type = answer_types[0] if answer_types else None
        
        category_instruction = CATEGORY_INSTRUCTIONS.get(primary_type, "")
        
        if category_instruction:
            full_prompt = f"""{context_text}

{category_instruction}

{MISSING_RULE}

Question: {qa.question}

Answer:"""
        else:
            full_prompt = f"""{context_text}

{MISSING_RULE}

Question: {qa.question}

Answer:"""

        meta = {
            "question_id": qa.metadata.get("question_id", ""),
            "answer_types": answer_types
        }
        return full_prompt, meta

    def post_process_answer(self, qa: StandardQA, raw_answer: str, meta: Dict[str, Any]) -> str:
        """
        Post-process raw answer generated by LLM.
        
        Current implementation only strips leading/trailing whitespace.
        
        Args:
            qa: Standardized QA object
            raw_answer: Raw answer generated by LLM
            meta: Metadata dictionary
            
        Returns:
            str: Processed answer
        """
        return raw_answer.strip()