LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
import logging
from executor.executors.exceptions import RetrievalError
from executor.executors.retrievers.base_retriever import BaseRetriever
from llama_index.core import VectorStoreIndex
from llama_index.core.indices.keyword_table import KeywordTableIndex
from llama_index.core.vector_stores import ExactMatchFilter, MetadataFilters
logger = logging.getLogger(__name__)
class KeywordTableRetriever(BaseRetriever):
"""Keyword table retrieval using LlamaIndex's native KeywordTableIndex."""
def retrieve(self) -> set[str]:
"""Retrieve text chunks using LlamaIndex's native KeywordTableIndex.
Returns:
set[str]: A set of text chunks retrieved from the database.
"""
try:
llm = self.require_llm()
logger.info(
f"Retrieving chunks for {self.doc_id} using LlamaIndex KeywordTableIndex."
)
# Get documents from vector index for keyword indexing
vector_store_index: VectorStoreIndex = self.vector_db.get_vector_store_index()
# Get all nodes for the document
all_retriever = vector_store_index.as_retriever(
similarity_top_k=1000, # Get all nodes
filters=MetadataFilters(
filters=[
ExactMatchFilter(key="doc_id", value=self.doc_id),
],
),
)
# Retrieve all nodes to build keyword index
all_nodes = all_retriever.retrieve(" ")
if not all_nodes:
logger.warning(f"No nodes found for doc_id: {self.doc_id}")
return set()
# Create KeywordTableIndex from nodes using our provided LLM
keyword_index = KeywordTableIndex(
nodes=[node.node for node in all_nodes],
show_progress=True,
llm=llm, # Use the provided LLM instead of defaulting to OpenAI
)
# Create retriever from keyword index
keyword_retriever = keyword_index.as_retriever(
similarity_top_k=self.top_k,
)
# Retrieve nodes using keyword matching
nodes = keyword_retriever.retrieve(self.prompt)
# Extract unique text chunks
chunks: set[str] = set()
for node in nodes:
chunks.add(node.get_content())
logger.info(
f"Successfully retrieved {len(chunks)} chunks using KeywordTableIndex."
)
return chunks
except (ValueError, AttributeError, KeyError, ImportError) as e:
logger.error(
"Error during keyword retrieval for %s: %s: %s",
self.doc_id,
type(e).__name__,
e,
exc_info=True,
)
raise RetrievalError(f"{type(e).__name__}: {e}") from e
except Exception as e:
logger.error(
"Unexpected error during keyword retrieval for %s: %s: %s",
self.doc_id,
type(e).__name__,
e,
exc_info=True,
)
raise RetrievalError(f"Unexpected error: {type(e).__name__}: {e}") from e