Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

3.2 KB
Raw Blame
import logging

from executor.executors.exceptions import RetrievalError
from executor.executors.retrievers.base_retriever import BaseRetriever
from llama_index.core import VectorStoreIndex
from llama_index.core.indices.keyword_table import KeywordTableIndex
from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters

logger = logging.getLogger(__name__)


class KeywordTableRetriever(BaseRetriever):
    """Keyword table retrieval using LlamaIndex's native KeywordTableIndex."""

    def retrieve(self) -> set[str]:
        """Retrieve text chunks using LlamaIndex's native KeywordTableIndex.

        Returns:
            set[str]: A set of text chunks retrieved from the database.
        """
        try:
            llm = self.require_llm()
            logger.info(
                f"Retrieving chunks for {self.doc_id} using LlamaIndex KeywordTableIndex."
            )

            # Get documents from vector index for keyword indexing
            vector_store_index: VectorStoreIndex = self.vector_db.get_vector_store_index()

            # Get all nodes for the document
            all_retriever = vector_store_index.as_retriever(
                similarity_top_k=1000,  # Get all nodes
                filters=MetadataFilters(
                    filters=[
                        ExactMatchFilter(key="doc_id", value=self.doc_id),
                    ],
                ),
            )

            # Retrieve all nodes to build keyword index
            all_nodes = all_retriever.retrieve(" ")

            if not all_nodes:
                logger.warning(f"No nodes found for doc_id: {self.doc_id}")
                return set()

            # Create KeywordTableIndex from nodes using our provided LLM
            keyword_index = KeywordTableIndex(
                nodes=[node.node for node in all_nodes],
                show_progress=True,
                llm=llm,  # Use the provided LLM instead of defaulting to OpenAI
            )

            # Create retriever from keyword index
            keyword_retriever = keyword_index.as_retriever(
                similarity_top_k=self.top_k,
            )

            # Retrieve nodes using keyword matching
            nodes = keyword_retriever.retrieve(self.prompt)

            # Extract unique text chunks
            chunks: set[str] = set()
            for node in nodes:
                chunks.add(node.get_content())

            logger.info(
                f"Successfully retrieved {len(chunks)} chunks using KeywordTableIndex."
            )
            return chunks

        except (ValueError, AttributeError, KeyError, ImportError) as e:
            logger.error(
                "Error during keyword retrieval for %s: %s: %s",
                self.doc_id,
                type(e).__name__,
                e,
                exc_info=True,
            )
            raise RetrievalError(f"{type(e).__name__}: {e}") from e
        except Exception as e:
            logger.error(
                "Unexpected error during keyword retrieval for %s: %s: %s",
                self.doc_id,
                type(e).__name__,
                e,
                exc_info=True,
            )
            raise RetrievalError(f"Unexpected error: {type(e).__name__}: {e}") from e