~95% on SimpleQA (e.g. Qwen3.6-27B on a 3090). Supports all local and cloud LLMs (llama.cpp, Ollama, Google, ...). 10+ search engines - arXiv, PubMed, your private documents. Everything Local & Encrypted.
Python academiaanthropicarxivbravedeep-researchencryptionhome-automationhomeserverlocallocal-deep-researchlocal-llmmistralollamaopenaipubmedresearchresearch-toolretrieval-augmented-generationsearxngself-hosted
"""
Elasticsearch utilities for indexing and managing documents.
"""
from loguru import logger
from pathlib import Path
from typing import Any, Dict, List, Optional
from elasticsearch import Elasticsearch
from elasticsearch.helpers import bulk
class ElasticsearchManager:
"""
Utility class for managing Elasticsearch indices and documents.
This class provides methods for creating indices, indexing documents,
and performing other Elasticsearch management tasks.
"""
def __init__(
self,
hosts: Optional[List[str]] = None,
username: Optional[str] = None,
password: Optional[str] = None,
api_key: Optional[str] = None,
cloud_id: Optional[str] = None,
):
"""
Initialize the Elasticsearch manager.
Args:
hosts: List of Elasticsearch hosts
username: Optional username for authentication
password: Optional password for authentication
api_key: Optional API key for authentication
cloud_id: Optional Elastic Cloud ID
"""
hosts = hosts or ["http://localhost:9200"]
# Initialize the Elasticsearch client
es_args: Dict[str, Any] = {}
# Basic authentication
if username and password:
es_args["basic_auth"] = (username, password)
# API key authentication
if api_key:
es_args["api_key"] = api_key
# Cloud ID for Elastic Cloud
if cloud_id:
es_args["cloud_id"] = cloud_id
# Connect to Elasticsearch
self.client = Elasticsearch(hosts, **es_args)
# Verify connection
try:
info = self.client.info()
logger.info(
f"Connected to Elasticsearch cluster: {info.get('cluster_name')}"
)
logger.info(
f"Elasticsearch version: {info.get('version', {}).get('number')}"
)
except Exception as e:
logger.exception("Failed to connect to Elasticsearch")
raise ConnectionError(f"Could not connect to Elasticsearch: {e!s}")
def close(self) -> None:
"""Close the Elasticsearch client and its connection pool."""
from .resource_utils import safe_close
safe_close(self.client, "Elasticsearch client")
def create_index(
self,
index_name: str,
mappings: Optional[Dict[str, Any]] = None,
settings: Optional[Dict[str, Any]] = None,
) -> bool:
"""
Create an Elasticsearch index with optional mappings and settings.
Args:
index_name: Name of the index to create
mappings: Optional mappings for the index fields
settings: Optional settings for the index
Returns:
bool: True if successful, False otherwise
"""
try:
# Check if index already exists
if self.client.indices.exists(index=index_name):
logger.warning(
f"Index '{index_name}' already exists - skipping creation"
)
return True
# Default mappings for better text search if none provided
if mappings is None:
mappings = {
"properties": {
"title": {
"type": "text",
"analyzer": "standard",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256,
}
},
},
"content": {"type": "text", "analyzer": "standard"},
"url": {"type": "keyword"},
"source": {"type": "keyword"},
"timestamp": {"type": "date"},
"metadata": {"type": "object", "enabled": True},
}
}
# Default settings if none provided
if settings is None:
settings = {
"number_of_shards": 1,
"number_of_replicas": 0,
"analysis": {
"analyzer": {"standard": {"type": "standard"}}
},
}
# Create the index with mappings and settings
create_response = self.client.indices.create(
index=index_name,
mappings=mappings,
settings=settings,
)
logger.info(f"Created index '{index_name}': {create_response}")
return True
except Exception:
logger.exception(f"Error creating index '{index_name}'")
return False
def delete_index(self, index_name: str) -> bool:
"""
Delete an Elasticsearch index.
Args:
index_name: Name of the index to delete
Returns:
bool: True if successful, False otherwise
"""
try:
# Check if index exists
if not self.client.indices.exists(index=index_name):
logger.warning(
f"Index '{index_name}' does not exist - skipping deletion"
)
return True
# Delete the index
delete_response = self.client.indices.delete(index=index_name)
logger.info(f"Deleted index '{index_name}': {delete_response}")
return True
except Exception:
logger.exception(f"Error deleting index '{index_name}'")
return False
def index_document(
self,
index_name: str,
document: Dict[str, Any],
document_id: Optional[str] = None,
refresh: bool = False,
) -> Optional[str]:
"""
Index a single document in Elasticsearch.
Args:
index_name: Name of the index to add the document to
document: The document to index
document_id: Optional document ID (will be generated if not provided)
refresh: Whether to refresh the index after indexing
Returns:
str: Document ID if successful, None otherwise
"""
try:
# Index the document
response = self.client.index(
index=index_name,
document=document,
id=document_id,
refresh=refresh,
)
logger.info(
f"Indexed document in '{index_name}' with ID: {response['_id']}"
)
return response["_id"]
except Exception:
logger.exception(f"Error indexing document in '{index_name}'")
return None
def bulk_index_documents(
self,
index_name: str,
documents: List[Dict[str, Any]],
id_field: Optional[str] = None,
refresh: bool = False,
) -> int:
"""
Bulk index multiple documents in Elasticsearch.
Args:
index_name: Name of the index to add the documents to
documents: List of documents to index
id_field: Optional field in the documents to use as the document ID
refresh: Whether to refresh the index after indexing
Returns:
int: Number of successfully indexed documents
"""
try:
# Prepare the bulk actions
actions = []
for doc in documents:
action = {
"_index": index_name,
"_source": doc,
}
# Use the specified field as the document ID if provided
if id_field and id_field in doc:
action["_id"] = doc[id_field]
actions.append(action)
# Execute the bulk indexing
success, failed = bulk(
self.client,
actions,
refresh=refresh,
stats_only=True,
)
logger.info(
f"Bulk indexed {success} documents in '{index_name}', failed: {failed}"
)
return success
except Exception:
logger.exception(f"Error bulk indexing documents in '{index_name}'")
return 0
def index_file(
self,
index_name: str,
file_path: str,
content_field: str = "content",
title_field: Optional[str] = "title",
extract_metadata: bool = True,
refresh: bool = False,
) -> Optional[str]:
"""
Index a file in Elasticsearch, extracting text content and metadata.
Args:
index_name: Name of the index to add the document to
file_path: Path to the file to index
content_field: Field name to store the file content
title_field: Field name to store the file title (filename if not specified)
extract_metadata: Whether to extract file metadata
refresh: Whether to refresh the index after indexing
Returns:
str: Document ID if successful, None otherwise
"""
try:
from unstructured.partition.auto import partition
# This is the same local partitioning + single-document join used
# by the deprecated LangChain loader, without constructing unused
# sync and async Unstructured API clients for every indexed file.
elements = partition(filename=file_path)
content = "\n\n".join(str(element) for element in elements)
# Get the filename for the title
filename = Path(file_path).name
title = filename
# Prepare the document
document = {
content_field: content,
}
# Add title if requested
if title_field:
document[title_field] = title
# Add metadata if requested
if extract_metadata:
# Legacy UnstructuredFileLoader(mode="single") exposed only
# file-level metadata. Preserve that behavior for empty files
# and multi-element documents alike.
document["metadata"] = {"source": file_path}
document["source"] = file_path
document["file_extension"] = Path(filename).suffix.lstrip(".")
document["filename"] = filename
# Index the document
return self.index_document(index_name, document, refresh=refresh)
except ImportError:
logger.error(
"Unstructured partitioner not available. Please install the "
"'unstructured' package."
)
return None
except Exception:
logger.exception(f"Error indexing file '{file_path}'")
return None
def index_directory(
self,
index_name: str,
directory_path: str,
file_patterns: Optional[List[str]] = None,
content_field: str = "content",
title_field: str = "title",
extract_metadata: bool = True,
refresh: bool = False,
) -> int:
"""
Index all matching files in a directory in Elasticsearch.
Args:
index_name: Name of the index to add the documents to
directory_path: Path to the directory containing files to index
file_patterns: List of file patterns to match (glob patterns)
content_field: Field name to store the file content
title_field: Field name to store the file title
extract_metadata: Whether to extract file metadata
refresh: Whether to refresh the index after indexing
Returns:
int: Number of successfully indexed files
"""
file_patterns = file_patterns or ["*.txt", "*.pdf", "*.docx", "*.md"]
try:
# Find all matching files
all_files = []
directory = Path(directory_path)
for pattern in file_patterns:
# nosemgrep: semgrep.rules.path-traversal-risk, reason: Local Python indexing API intentionally accepts caller-selected directories and glob patterns; no web caller
matching_files = list(directory.glob(pattern))
all_files.extend([str(f) for f in matching_files])
logger.info(
f"Found {len(all_files)} files matching patterns {file_patterns} in {directory_path}"
)
# Index each file
successful_count = 0
for file_path in all_files:
logger.info(f"Indexing file: {file_path}")
doc_id = self.index_file(
index_name=index_name,
file_path=file_path,
content_field=content_field,
title_field=title_field,
extract_metadata=extract_metadata,
refresh=refresh,
)
if doc_id:
successful_count += 1
logger.info(
f"Successfully indexed {successful_count} files out of {len(all_files)}"
)
return successful_count
except Exception:
logger.exception(f"Error indexing directory '{directory_path}'")
return 0
def search(
self,
index_name: str,
query: str,
fields: List[str] = ["content", "title"],
size: int = 10,
highlight: bool = True,
) -> Dict[str, Any]:
"""
Search for documents in Elasticsearch.
Args:
index_name: Name of the index to search
query: Search query
fields: Fields to search in
size: Maximum number of results to return
highlight: Whether to include highlighted excerpts in results
Returns:
Dict: Elasticsearch search response
"""
try:
search_query = {
"query": {
"multi_match": {
"query": query,
"fields": fields,
"type": "best_fields",
"tie_breaker": 0.3,
}
},
"size": size,
}
# Add highlighting if requested
if highlight:
search_query["highlight"] = {
"fields": {field: {} for field in fields},
"pre_tags": ["<em>"],
"post_tags": ["</em>"],
}
# Execute the search
return self.client.search(
index=index_name,
body=search_query,
)
except Exception:
logger.exception(f"Error searching index '{index_name}'")
return {"error": "Search failed"}