Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
9.2 KB
Raw Blame
import os
import time
from typing import Any

from utils.logging_config import get_logger

from .processors import TaskProcessor
from .tasks import FileTask, TaskStatus, UploadTask

logger = get_logger(__name__)


class UrlProcessor(TaskProcessor):
    """Processor for URL ingestion using the traditional (non-Langflow) pipeline."""

    def __init__(
        self,
        document_service,
        models_service,
        docs_url: str,
        crawl_depth: int,
        owner_user_id: str = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        connector_type: str = "openrag_docs",
        is_sample_data: bool = False,
    ):
        super().__init__(document_service=document_service, models_service=models_service)
        self.docs_url = docs_url
        self.crawl_depth = crawl_depth
        self.owner_user_id = owner_user_id
        self.jwt_token = jwt_token
        self.owner_name = owner_name
        self.owner_email = owner_email
        self.connector_type = connector_type
        self.is_sample_data = is_sample_data

    async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
        from utils.hash_utils import hash_id
        from utils.url_content_fetcher import materialize_url_as_text_file

        from .processors import DocumentFileProcessor

        file_task.status = TaskStatus.RUNNING
        file_task.updated_at = time.time()

        temp_file_path = None
        try:
            temp_file_path, title = await materialize_url_as_text_file(
                docs_url=self.docs_url,
                crawl_depth=self.crawl_depth,
            )

            processor = DocumentFileProcessor(
                self.document_service,
                models_service=self.models_service,
                owner_user_id=self.owner_user_id,
                jwt_token=self.jwt_token,
                owner_name=self.owner_name,
                owner_email=self.owner_email,
                is_sample_data=self.is_sample_data,
                connector_type=self.connector_type,
                # Only process_document_standard runs on this instance; the
                # duplicate gate, and so the delete, never does.
                allow_anonymous_delete=False,
            )
            result = await processor.process_document_standard(
                file_path=temp_file_path,
                file_hash=hash_id(temp_file_path),
                owner_user_id=self.owner_user_id,
                original_filename=title,
                jwt_token=self.jwt_token,
                owner_name=self.owner_name,
                owner_email=self.owner_email,
                file_size=os.path.getsize(temp_file_path),
                connector_type=self.connector_type,
                is_sample_data=self.is_sample_data,
            )

            if result.get("status") == "error":
                file_task.status = TaskStatus.FAILED
                file_task.error = result.get("error") or "Failed to process document"
                file_task.updated_at = time.time()
                upload_task.failed_files += 1
            else:
                file_task.status = TaskStatus.COMPLETED
                file_task.result = result
                file_task.updated_at = time.time()
                upload_task.successful_files += 1

        except Exception as e:
            file_task.status = TaskStatus.FAILED
            file_task.error = str(e)
            file_task.updated_at = time.time()
            upload_task.failed_files += 1
            raise
        finally:
            if temp_file_path:
                try:
                    os.unlink(temp_file_path)
                except FileNotFoundError:
                    pass
                except Exception as e:
                    logger.error(
                        "Failed to clean temporary URL docs file",
                        path=temp_file_path,
                        error=str(e),
                    )


class LangflowUrlProcessor(TaskProcessor):
    """Processor for Langflow URL ingestion flow as a tracked task."""

    def __init__(
        self,
        langflow_file_service,
        session_manager,
        docs_url: str,
        crawl_depth: int,
        owner_user_id: str = None,
        jwt_token: str = None,
        owner_name: str = None,
        owner_email: str = None,
        connector_type: str = "openrag_docs",
        prevent_outside: bool = True,
        tweaks: dict[str, Any] | None = None,
    ):
        super().__init__()
        self.langflow_file_service = langflow_file_service
        self.session_manager = session_manager
        self.docs_url = docs_url
        self.crawl_depth = crawl_depth
        self.owner_user_id = owner_user_id
        self.jwt_token = jwt_token
        self.owner_name = owner_name
        self.owner_email = owner_email
        self.connector_type = connector_type
        self.prevent_outside = prevent_outside
        self.tweaks = tweaks

    async def _count_system_default_docs(self, opensearch_client) -> int:
        """Count indexed OpenRAG docs for the current owner/context."""
        from config.settings import get_index_name

        must_filters = [
            {
                "bool": {
                    "should": [
                        {"term": {"connector_type.keyword": self.connector_type}},
                        {"term": {"connector_type": self.connector_type}},
                    ],
                    "minimum_should_match": 1,
                }
            }
        ]
        if self.owner_email:
            must_filters.append(
                {
                    "bool": {
                        "should": [
                            {"term": {"owner_email.keyword": self.owner_email}},
                            {"match_phrase": {"owner_email": self.owner_email}},
                        ],
                        "minimum_should_match": 1,
                    }
                }
            )
        if self.owner_user_id:
            must_filters.append(
                {
                    "bool": {
                        "should": [
                            {"term": {"owner.keyword": self.owner_user_id}},
                            {"match_phrase": {"owner": self.owner_user_id}},
                        ],
                        "minimum_should_match": 1,
                    }
                }
            )

        count_query = {"query": {"bool": {"must": must_filters}}}
        response = await opensearch_client.count(
            index=get_index_name(),
            body=count_query,
        )
        return int(response.get("count", 0))

    async def process_item(self, upload_task: UploadTask, item: str, file_task: FileTask) -> None:
        """Process one URL ingestion item through Langflow."""
        file_task.status = TaskStatus.RUNNING
        file_task.updated_at = time.time()

        try:
            effective_jwt = self.jwt_token
            if self.session_manager and not effective_jwt:
                effective_jwt = self.session_manager.get_effective_jwt_token(
                    self.owner_user_id,
                    None,
                )

            opensearch_client = self.session_manager.get_user_opensearch_client(
                self.owner_user_id, effective_jwt
            )
            docs_before = await self._count_system_default_docs(opensearch_client)
            logger.info(
                "URL ingestion before",
                docs_before=docs_before,
            )

            result = await self.langflow_file_service.run_url_ingestion_flow(
                docs_url=self.docs_url,
                crawl_depth=self.crawl_depth,
                jwt_token=effective_jwt,
                owner=self.owner_user_id,
                owner_name=self.owner_name,
                owner_email=self.owner_email,
                connector_type=self.connector_type,
                prevent_outside=self.prevent_outside,
                tweaks=self.tweaks,
            )
            # docs_after = await self._count_system_default_docs(opensearch_client)
            # logger.info(
            #     "URL ingestion after",
            #     docs_after=docs_after,
            # )
            # # OpenSearch visibility can lag indexing by a refresh interval.
            # # Retry a few times before declaring ingestion failed.
            # for _ in range(5):
            #     if docs_after > 0:
            #         break
            #     await asyncio.sleep(1)
            #     docs_after = await self._count_system_default_docs(opensearch_client)
            # if docs_after <= 0:
            #     raise ValueError(
            #         "URL ingestion completed but no OpenRAG docs were indexed"
            #     )
            # logger.info(
            #     "URL ingestion indexed OpenRAG docs",
            #     docs_before=docs_before,
            #     docs_after=docs_after,
            # )

            file_task.status = TaskStatus.COMPLETED
            file_task.result = result
            file_task.updated_at = time.time()
            upload_task.successful_files += 1

        except Exception as e:
            file_task.status = TaskStatus.FAILED
            file_task.error = str(e)
            file_task.updated_at = time.time()
            upload_task.failed_files += 1
            raise