Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

3.9 KB
Raw Blame
import logging

from executor.executors.exceptions import RetrievalError
from executor.executors.retrievers.base_retriever import BaseRetriever
from llama_index.core import VectorStoreIndex
from llama_index.core.retrievers import QueryFusionRetriever
from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters

logger = logging.getLogger(__name__)


class FusionRetriever(BaseRetriever):
    """Fusion retrieval class using LlamaIndex's native QueryFusionRetriever.

    This technique generates multiple query variations and combines results
    using reciprocal rank fusion for improved relevance.
    """

    def retrieve(self) -> set[str]:
        """Retrieve text chunks using LlamaIndex's QueryFusionRetriever.

        Returns:
            set[str]: A set of text chunks retrieved from the database.
        """
        try:
            llm = self.require_llm()
            logger.info(
                f"Retrieving chunks for {self.doc_id} using LlamaIndex QueryFusionRetriever."
            )

            # Get the vector store index
            vector_store_index: VectorStoreIndex = self.vector_db.get_vector_store_index()

            # Create multiple retrievers with different parameters for true fusion
            filters = MetadataFilters(
                filters=[
                    ExactMatchFilter(key="doc_id", value=self.doc_id),
                ],
            )

            # Retriever 1: Standard similarity search
            retriever_1 = vector_store_index.as_retriever(
                similarity_top_k=self.top_k,
                filters=filters,
            )

            # Retriever 2: Broader search with more candidates
            retriever_2 = vector_store_index.as_retriever(
                similarity_top_k=self.top_k * 2,
                filters=filters,
            )

            # Retriever 3: Focused search with fewer candidates
            retriever_3 = vector_store_index.as_retriever(
                similarity_top_k=max(1, self.top_k // 2),
                filters=filters,
            )

            # Create LlamaIndex QueryFusionRetriever with multiple retrievers
            fusion_retriever = QueryFusionRetriever(
                [retriever_1, retriever_2, retriever_3],  # Multiple retrievers for fusion
                similarity_top_k=self.top_k,
                num_queries=4,  # Generate multiple query variations
                mode="simple",  # Use simple fusion mode (reciprocal rank fusion)
                use_async=False,
                verbose=True,
                llm=llm,  # LLM generates query variations
            )

            # Retrieve nodes using fusion technique
            nodes = fusion_retriever.retrieve(self.prompt)

            # Extract unique text chunks
            chunks: set[str] = set()
            for node in nodes:
                if node.score > 0:
                    chunks.add(node.get_content())
                else:
                    logger.info(
                        f"Node score is less than 0. "
                        f"Ignored: {node.node_id} with score {node.score}"
                    )

            logger.info(f"Successfully retrieved {len(chunks)} chunks using fusion.")
            return chunks

        except (ValueError, AttributeError, KeyError, ImportError) as e:
            logger.error(
                "Error during fusion retrieval for %s: %s: %s",
                self.doc_id,
                type(e).__name__,
                e,
                exc_info=True,
            )
            raise RetrievalError(f"{type(e).__name__}: {e}") from e
        except Exception as e:
            logger.error(
                "Unexpected error during fusion retrieval for %s: %s: %s",
                self.doc_id,
                type(e).__name__,
                e,
                exc_info=True,
            )
            raise RetrievalError(f"Unexpected error: {type(e).__name__}: {e}") from e