OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
import tiktoken
from config.settings import get_embedding_model, get_index_name
from utils.document_processing import extract_relevant
from utils.logging_config import get_logger
logger = get_logger(__name__)
def get_token_count(text: str, model: str = None) -> int:
"""Get accurate token count using tiktoken"""
model = model or get_embedding_model()
try:
encoding = tiktoken.encoding_for_model(model)
return len(encoding.encode(text))
except KeyError:
# Fallback to cl100k_base for unknown models
encoding = tiktoken.get_encoding("cl100k_base")
return len(encoding.encode(text))
def chunk_texts_for_embeddings(
texts: list[str], max_tokens: int = None, model: str = None
) -> list[list[str]]:
"""Pack texts into batches that stay within ``max_tokens``.
Each input stays one embedding input. A text already over the limit is
placed in its own batch; splitting it here would make the embedding list
longer than the chunk list. If ``max_tokens`` is None, texts are returned
as a single batch.
"""
model = model or get_embedding_model()
# Filter out empty or whitespace-only texts
texts = [t for t in texts if t and t.strip()]
if not texts:
return []
if max_tokens is None:
return [texts]
batches = []
current_batch: list[str] = []
current_tokens = 0
for text in texts:
text_tokens = get_token_count(text, model)
over_limit = text_tokens > max_tokens
if over_limit or (current_batch and current_tokens + text_tokens > max_tokens):
if current_batch:
batches.append(current_batch)
current_batch = []
current_tokens = 0
if over_limit:
batches.append([text])
continue
current_batch.append(text)
current_tokens += text_tokens
if current_batch:
batches.append(current_batch)
return batches
class DocumentService:
def __init__(
self,
session_manager=None,
models_service=None,
docling_service=None,
document_index_writer=None,
):
self.session_manager = session_manager
self.models_service = models_service
self.docling_service = docling_service
self.document_index_writer = document_index_writer
self._mapping_ensured = False
async def process_upload_file(
self,
upload_file,
owner_user_id: str = None,
jwt_token: str = None,
owner_name: str = None,
owner_email: str = None,
):
"""Process an uploaded file from form data"""
import os
# Default metadata for anonymous users if not provided
from session_manager import AnonymousUser
from utils.file_utils import auto_cleanup_tempfile
from utils.hash_utils import hash_id
anonymous_user = AnonymousUser()
owner_name = owner_name or anonymous_user.name
owner_email = owner_email or anonymous_user.email
# Preserve file extension for docling format detection
filename = upload_file.filename or "uploaded"
suffix = os.path.splitext(filename)[1] or ""
with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
# Stream upload file to temporary file
file_size = 0
with open(tmp_path, "wb") as tmp_file:
while True:
chunk = await upload_file.read(1 << 20)
if not chunk:
break
tmp_file.write(chunk)
file_size += len(chunk)
file_hash = hash_id(tmp_path)
# Get user's OpenSearch client with JWT for OIDC auth
opensearch_client = self.session_manager.get_user_opensearch_client(
owner_user_id, jwt_token
)
try:
exists = await opensearch_client.exists(index=get_index_name(), id=file_hash)
except Exception as e:
logger.error("OpenSearch exists check failed", file_hash=file_hash, error=str(e))
raise
if exists:
return {"status": "unchanged", "id": file_hash}
# Use consolidated standard processing
from models.processors import TaskProcessor
processor = TaskProcessor(
document_service=self,
models_service=self.models_service,
docling_service=self.docling_service,
)
result = await processor.process_document_standard(
file_path=tmp_path,
file_hash=file_hash,
owner_user_id=owner_user_id,
original_filename=upload_file.filename,
jwt_token=jwt_token,
owner_name=owner_name,
owner_email=owner_email,
file_size=file_size,
connector_type="local",
)
return result
async def process_upload_context(
self, upload_file, filename: str = None, user_id: str = None, jwt_token: str = None
):
"""Process uploaded file and return content for context"""
import io
import os
if not filename:
filename = upload_file.filename or "uploaded_document"
from utils.upload_limits import read_upload_within_limit
# Stop once the file crosses the upload limit, before the rest is buffered.
content = io.BytesIO(await read_upload_within_limit(upload_file, filename))
# Check if this is a .txt or .md file - use simple processing
file_ext = os.path.splitext(filename)[1].lower()
if file_ext in (".txt", ".md"):
# Simple text file processing for chat context
text_content = content.read().decode("utf-8", errors="replace")
# For context, we don't need to chunk - just return the full content
return {
"filename": filename,
"content": text_content,
"pages": 1, # Text files don't have pages
"content_length": len(text_content),
}
else:
full_doc = await self.docling_service.convert_bytes(
content.read(), filename, user_id=user_id, auth_header=jwt_token
)
slim_doc = extract_relevant(full_doc)
# Extract all text content
all_text = []
for chunk in slim_doc["chunks"]:
all_text.append(f"Page {chunk['page']}:\n{chunk['text']}")
full_content = "\n\n".join(all_text)
return {
"filename": filename,
"content": full_content,
"pages": len(slim_doc["chunks"]),
"content_length": len(full_content),
}