Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
7.6 KB
Raw Blame
import asyncio
import os
from pathlib import Path
from unittest.mock import AsyncMock, patch

import httpx
import pytest


# Pop cached modules so they reload with modified env vars
def _purge_modules():
    import sys

    for mod in [
        "api.router",
        "api.connector_router",
        "config.config_manager",  # reset cached OpenRAGConfig so env-var overrides take effect
        "config.settings",
        "auth_middleware",
        "main",
        "api",
        "services",
        "services.search_service",
        "services.default_docs_service",
        "services.startup_orchestrator",
        "app.routes.internal",
        "app.routes",
        "app.container",
        "app.factory",
        "app.lifespan",
        "dependencies",
        "utils.opensearch_init",
    ]:
        sys.modules.pop(mod, None)


async def wait_for_service_ready(client: httpx.AsyncClient, timeout_s: float = 30.0):
    deadline = asyncio.get_event_loop().time() + timeout_s
    last_err = None
    while asyncio.get_event_loop().time() < deadline:
        try:
            r1 = await client.get("/auth/me")
            if r1.status_code != 200:
                await asyncio.sleep(0.5)
                continue
            r2 = await client.post("/search", json={"query": "*", "limit": 0})
            if r2.status_code == 200:
                return
            last_err = r2.text
        except Exception as e:
            last_err = str(e)
        await asyncio.sleep(0.5)
    raise AssertionError(f"Service not ready in time: {last_err}")


async def wait_for_task_completion(
    client: httpx.AsyncClient, task_id: str, timeout_s: float = 60.0
) -> dict:
    deadline = asyncio.get_event_loop().time() + timeout_s
    last_payload = None
    while asyncio.get_event_loop().time() < deadline:
        resp = await client.get(f"/tasks/{task_id}")
        if resp.status_code == 200:
            try:
                data = resp.json()
            except Exception:
                last_payload = resp.text
            else:
                status = (data.get("status") or "").lower()
                if status == "completed":
                    return data
                if status == "failed":
                    raise AssertionError(f"Task {task_id} failed: {data}")
                last_payload = data
        await asyncio.sleep(1.0)
    raise AssertionError(f"Task {task_id} did not complete in time. Last payload: {last_payload}")


@pytest.mark.asyncio
async def test_non_langflow_csv_ingestion_with_splitting(tmp_path: Path):
    """Validate standard CSV ingestion using standard non-Langflow processing pipeline.

    Simulates parsing of a CSV file yielding a large table chunk (exceeding standard 8,000 token limit),
    verifying it gets split correctly and indexed into OpenSearch.
    """
    os.environ["DISABLE_INGEST_WITH_LANGFLOW"] = "true"
    os.environ["DISABLE_STARTUP_INGEST"] = "true"
    os.environ["EMBEDDING_MODEL"] = os.getenv("EMBEDDING_MODEL", "text-embedding-3-small")
    azure_key = os.getenv("AZURE_OPENAI_API_KEY") or os.getenv("AZURE_API_KEY")
    azure_endpoint = (
        os.getenv("AZURE_OPENAI_ENDPOINT")
        or os.getenv("AZURE_OPENAI_API_BASE")
        or os.getenv("AZURE_API_BASE")
    )
    os.environ["EMBEDDING_PROVIDER"] = os.getenv(
        "EMBEDDING_PROVIDER",
        "azure" if (azure_key and azure_endpoint) else "openai",
    )
    os.environ["GOOGLE_OAUTH_CLIENT_ID"] = ""
    os.environ["GOOGLE_OAUTH_CLIENT_SECRET"] = ""

    _purge_modules()

    from config.settings import clients, get_index_name
    from main import create_app, startup_tasks

    await clients.initialize()
    try:
        await clients.opensearch.indices.delete(index=get_index_name())
        await asyncio.sleep(1)
    except Exception:
        pass

    app = await create_app()
    await startup_tasks(app.state.services)

    from main import _ensure_opensearch_index

    await _ensure_opensearch_index()

    # Mock the DoclingService conversion result
    # We want a table chunk whose token count exceeds 8,000 tokens (so it gets split)
    # "testlargephrase " is 3 tokens in cl100k_base. Repeating it 4000 times gives ~12,000 tokens.
    large_table_text = "testlargephrase " * 4000

    mock_docling_result = {
        "origin": {
            "binary_hash": "sha-csv-integration-123",
            "filename": "correlation_report.csv",
            "mimetype": "text/csv",
        },
        "texts": [],
        "tables": [
            {
                "prov": [{"page_no": 1}],
                "data": {
                    "table_cells": [
                        {
                            "start_row_offset_idx": 0,
                            "start_col_offset_idx": 0,
                            "text": large_table_text,
                        }
                    ]
                },
            }
        ],
    }

    # Patch convert_file method on DoclingService class
    from services.docling_service import DoclingService

    with patch.object(DoclingService, "convert_file", AsyncMock(return_value=mock_docling_result)):
        transport = httpx.ASGITransport(app=app)
        async with httpx.AsyncClient(transport=transport, base_url="http://testserver") as client:
            await wait_for_service_ready(client)

            # Create mock CSV
            csv_path = tmp_path / "correlation_report.csv"
            csv_path.write_text("header1,header2\nvalue1,value2")

            # DocumentFileProcessor computes document_id as hash_id(file_path), not
            # from Docling's binary_hash field, so derive the expected id here.
            from utils.hash_utils import hash_id as _hash_id

            expected_document_id = _hash_id(str(csv_path))

            files = {
                "file": (
                    csv_path.name,
                    csv_path.read_bytes(),
                    "text/csv",
                )
            }

            resp = await client.post("/router/upload_ingest", files=files)
            assert resp.status_code == 202, resp.text

            data = resp.json()
            task_id = data.get("task_id")
            assert task_id is not None

            # Wait for processing to complete
            await wait_for_task_completion(client, task_id)

            # Wait for search indices to refresh
            await asyncio.sleep(1)

            # Retrieve results from OpenSearch using search endpoint
            search_resp = await client.post(
                "/search",
                json={"query": "testlargephrase", "limit": 10},
            )
            assert search_resp.status_code == 200, search_resp.text

            results = search_resp.json().get("results", [])
            assert len(results) > 0, "No search results returned from indexed chunks"

            # Check direct OpenSearch doc count for this document ID
            opensearch_resp = await clients.opensearch.search(
                index=get_index_name(),
                body={"query": {"term": {"document_id": expected_document_id}}},
            )
            hits = opensearch_resp.get("hits", {}).get("hits", [])

            # Since the text is ~12,000 tokens, and max_tokens=1000,
            # it should be split into 10 chunks, resulting in 10 documents in OpenSearch.
            assert len(hits) == 10, f"Expected 10 indexed chunks, but got {len(hits)}"
            for hit in hits:
                source = hit.get("_source", {})
                assert source.get("document_id") == expected_document_id
                assert source.get("filename") == "correlation_report.csv"
                assert source.get("mimetype") == "text/csv"
                assert "testlargephrase" in source.get("text", "")