Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
6.7 KB
Raw Blame
import tiktoken

from config.settings import get_embedding_model, get_index_name
from utils.document_processing import extract_relevant
from utils.logging_config import get_logger

logger = get_logger(__name__)


def get_token_count(text: str, model: str = None) -> int:
    """Get accurate token count using tiktoken"""
    model = model or get_embedding_model()
    try:
        encoding = tiktoken.encoding_for_model(model)
        return len(encoding.encode(text))
    except KeyError:
        # Fallback to cl100k_base for unknown models
        encoding = tiktoken.get_encoding("cl100k_base")
        return len(encoding.encode(text))


def chunk_texts_for_embeddings(
    texts: list[str], max_tokens: int = None, model: str = None
) -> list[list[str]]:
    """Pack texts into batches that stay within ``max_tokens``.

    Each input stays one embedding input. A text already over the limit is
    placed in its own batch; splitting it here would make the embedding list
    longer than the chunk list. If ``max_tokens`` is None, texts are returned
    as a single batch.
    """
    model = model or get_embedding_model()

    # Filter out empty or whitespace-only texts
    texts = [t for t in texts if t and t.strip()]

    if not texts:
        return []

    if max_tokens is None:
        return [texts]

    batches = []
    current_batch: list[str] = []
    current_tokens = 0

    for text in texts:
        text_tokens = get_token_count(text, model)

        over_limit = text_tokens > max_tokens
        if over_limit or (current_batch and current_tokens + text_tokens > max_tokens):
            if current_batch:
                batches.append(current_batch)
                current_batch = []
                current_tokens = 0
            if over_limit:
                batches.append([text])
                continue

        current_batch.append(text)
        current_tokens += text_tokens

    if current_batch:
        batches.append(current_batch)

    return batches


class DocumentService:
    def __init__(
        self,
        session_manager=None,
        models_service=None,
        docling_service=None,
        document_index_writer=None,
    ):
        self.session_manager = session_manager
        self.models_service = models_service
        self.docling_service = docling_service
        self.document_index_writer = document_index_writer
        self._mapping_ensured = False

    async def process_upload_file(
        self,
        upload_file,
        owner_user_id: str = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
    ):
        """Process an uploaded file from form data"""
        import os

        # Default metadata for anonymous users if not provided
        from session_manager import AnonymousUser
        from utils.file_utils import auto_cleanup_tempfile
        from utils.hash_utils import hash_id

        anonymous_user = AnonymousUser()
        owner_name = owner_name or anonymous_user.name
        owner_email = owner_email or anonymous_user.email

        # Preserve file extension for docling format detection
        filename = upload_file.filename or "uploaded"
        suffix = os.path.splitext(filename)[1] or ""

        with auto_cleanup_tempfile(suffix=suffix) as tmp_path:
            # Stream upload file to temporary file
            file_size = 0
            with open(tmp_path, "wb") as tmp_file:
                while True:
                    chunk = await upload_file.read(1 << 20)
                    if not chunk:
                        break
                    tmp_file.write(chunk)
                    file_size += len(chunk)

            file_hash = hash_id(tmp_path)
            # Get user's OpenSearch client with JWT for OIDC auth
            opensearch_client = self.session_manager.get_user_opensearch_client(
                owner_user_id, jwt_token
            )

            try:
                exists = await opensearch_client.exists(index=get_index_name(), id=file_hash)
            except Exception as e:
                logger.error("OpenSearch exists check failed", file_hash=file_hash, error=str(e))
                raise
            if exists:
                return {"status": "unchanged", "id": file_hash}

            # Use consolidated standard processing
            from models.processors import TaskProcessor

            processor = TaskProcessor(
                document_service=self,
                models_service=self.models_service,
                docling_service=self.docling_service,
            )
            result = await processor.process_document_standard(
                file_path=tmp_path,
                file_hash=file_hash,
                owner_user_id=owner_user_id,
                original_filename=upload_file.filename,
                jwt_token=jwt_token,
                owner_name=owner_name,
                owner_email=owner_email,
                file_size=file_size,
                connector_type="local",
            )
            return result

    async def process_upload_context(
        self, upload_file, filename: str = None, user_id: str = None, jwt_token: str = None
    ):
        """Process uploaded file and return content for context"""
        import io
        import os

        if not filename:
            filename = upload_file.filename or "uploaded_document"

        from utils.upload_limits import read_upload_within_limit

        # Stop once the file crosses the upload limit, before the rest is buffered.
        content = io.BytesIO(await read_upload_within_limit(upload_file, filename))

        # Check if this is a .txt or .md file - use simple processing
        file_ext = os.path.splitext(filename)[1].lower()

        if file_ext in (".txt", ".md"):
            # Simple text file processing for chat context
            text_content = content.read().decode("utf-8", errors="replace")

            # For context, we don't need to chunk - just return the full content
            return {
                "filename": filename,
                "content": text_content,
                "pages": 1,  # Text files don't have pages
                "content_length": len(text_content),
            }
        else:
            full_doc = await self.docling_service.convert_bytes(
                content.read(), filename, user_id=user_id, auth_header=jwt_token
            )
            slim_doc = extract_relevant(full_doc)

            # Extract all text content
            all_text = []
            for chunk in slim_doc["chunks"]:
                all_text.append(f"Page {chunk['page']}:\n{chunk['text']}")

            full_content = "\n\n".join(all_text)

            return {
                "filename": filename,
                "content": full_content,
                "pages": len(slim_doc["chunks"]),
                "content_length": len(full_content),
            }