Skip to content

LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows

2.8 KB
Raw Blame
import logging

from executor.executors.exceptions import RetrievalError
from executor.executors.retrievers.base_retriever import BaseRetriever
from llama_index.core.query_engine import SubQuestionQueryEngine
from llama_index.core.question_gen.llm_generators import LLMQuestionGenerator
from llama_index.core.schema import QueryBundle
from llama_index.core.tools import QueryEngineTool, ToolMetadata

logger = logging.getLogger(__name__)


class SubquestionRetriever(BaseRetriever):
    """SubquestionRetrieval class for querying VectorDB using LlamaIndex's
    SubQuestionQueryEngine.
    """

    def retrieve(self) -> set[str]:
        """Retrieve text chunks from the VectorDB based on the provided prompt.

        Returns:
            set[str]: A set of text chunks retrieved from the database.
        """
        try:
            llm = self.require_llm()
            logger.info("Initialising vector query engine...")
            vector_query_engine = self.vector_db.get_vector_store_index().as_query_engine(
                llm=llm, similarity_top_k=self.top_k
            )
            logger.info(
                f"Retrieving chunks for {self.doc_id} using SubQuestionQueryEngine."
            )
            query_engine_tools = [
                QueryEngineTool(
                    query_engine=vector_query_engine,
                    metadata=ToolMetadata(
                        name=self.doc_id, description=f"Nodes for {self.doc_id}"
                    ),
                ),
            ]
            query_bundle = QueryBundle(query_str=self.prompt)

            question_gen = LLMQuestionGenerator.from_defaults(
                llm=llm,
            )
            query_engine = SubQuestionQueryEngine.from_defaults(
                query_engine_tools=query_engine_tools,
                question_gen=question_gen,
                use_async=True,
                llm=llm,
            )

            response = query_engine.query(str_or_query_bundle=query_bundle)

            chunks: set[str] = {node.text for node in response.source_nodes}
            logger.info(f"Successfully retrieved {len(chunks)} chunks.")
            return chunks

        except (ValueError, AttributeError, KeyError, ImportError) as e:
            logger.error(
                "Error during retrieving chunks %s: %s: %s",
                self.doc_id,
                type(e).__name__,
                e,
                exc_info=True,
            )
            raise RetrievalError(f"{type(e).__name__}: {e}") from e
        except Exception as e:
            logger.error(
                "Unexpected error during retrieving chunks %s: %s: %s",
                self.doc_id,
                type(e).__name__,
                e,
                exc_info=True,
            )
            raise RetrievalError(f"Unexpected error: {type(e).__name__}: {e}") from e