Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
12.1 KB
Raw Blame
"""One Langfuse trace per file on the non-Langflow ingest path (issue 03).

`process_document_standard` runs against fakes for Docling, the gateway and
OpenSearch; the spans go through the real Langfuse SDK into memory.
"""

from unittest.mock import AsyncMock, MagicMock

import pytest
from langfuse._client.attributes import LangfuseOtelSpanAttributes as LF
from opentelemetry import trace

from observability import spans
from observability.tracing import trace_context

STAGES = {
    "ingest.parse",
    "ingest.extract",
    "ingest.resplit",
    "ingest.token_cap",
    "ingest.embed",
    "ingest.index",
}


def _metadata(span, key):
    return span.attributes.get(f"{LF.OBSERVATION_METADATA}.openrag.{key}")


def _trace_metadata(span, key):
    return span.attributes.get(f"{LF.TRACE_METADATA}.openrag.{key}")


DOCLING_DOC = {
    "origin": {"filename": "report.pdf", "mimetype": "application/pdf"},
    "pages": {"1": {}, "2": {}},
    "texts": [
        {"text": "Intro paragraph", "prov": [{"page_no": 1}]},
        {"text": "Page two text", "prov": [{"page_no": 2}]},
        {"text": "More on two", "prov": [{"page_no": 2}]},
    ],
    "tables": [
        {
            "prov": [{"page_no": 2}],
            "data": {
                "table_cells": [
                    {"start_row_offset_idx": 0, "start_col_offset_idx": 0, "text": "a"},
                ]
            },
        }
    ],
    "pictures": [{"prov": [{"page_no": 1}], "annotations": []}],
}


class _Pipeline:
    """`process_document_standard` with every external dependency faked."""

    def __init__(self, monkeypatch, *, exists=False, stale_ids=("doc-1_0", "doc-1_1")):
        from models import processors as processors_mod
        from models.processors import TaskProcessor
        from services import llm_gateway as gateway_mod
        from utils import opensearch_delete

        self.opensearch = AsyncMock()
        self.opensearch.exists = AsyncMock(return_value=exists)
        self.check_exists = exists

        document_service = MagicMock()
        document_service.session_manager.get_user_opensearch_client = MagicMock(
            return_value=self.opensearch
        )
        self.indexed: list = []

        class _Writer:
            async def index_chunks(writer_self, context, chunks, *, final=False):
                self.indexed.extend(chunks)

        document_service.document_index_writer = _Writer()

        models_service = MagicMock()
        models_service.get_litellm_model_name = AsyncMock(return_value="text-embedding-3-small")

        self.docling = MagicMock()
        self.docling.convert_file = AsyncMock(return_value=DOCLING_DOC)

        self.processor = TaskProcessor(
            document_service=document_service,
            models_service=models_service,
            docling_service=self.docling,
        )
        self.processor.check_document_exists = AsyncMock(return_value=exists)

        config = MagicMock()
        config.knowledge.embedding_model = "text-embedding-3-small"
        config.knowledge.embedding_provider = "openai"
        config.knowledge.chunk_size = None
        config.knowledge.chunk_overlap = None
        config.knowledge.ocr = True
        config.knowledge.picture_descriptions = False
        config.knowledge.table_structure = True
        monkeypatch.setattr(processors_mod, "get_openrag_config", lambda: config)
        monkeypatch.setattr(processors_mod, "get_index_name", lambda: "test-index")
        fake_clients = MagicMock()
        fake_clients.opensearch = AsyncMock()
        monkeypatch.setattr(processors_mod, "clients", fake_clients)

        async def _collect(*_args, **_kwargs):
            return list(stale_ids)

        async def _delete(*_args, **_kwargs):
            return None

        monkeypatch.setattr(opensearch_delete, "collect_visible_document_ids", _collect)
        monkeypatch.setattr(opensearch_delete, "delete_document_ids", _delete)

        self.embed_parents: list[str] = []
        self.embed_error: Exception | None = None

        async def _embeddings(body, **_kwargs):
            self.embed_parents.append(getattr(trace.get_current_span(), "name", None))
            if self.embed_error is not None:
                raise self.embed_error
            return {
                "data": [{"embedding": [0.1, 0.2, 0.3]} for _ in body["input"]],
                "usage": {"prompt_tokens": 5 * len(body["input"]), "total_tokens": 0},
            }

        monkeypatch.setattr(gateway_mod, "embeddings", _embeddings)

    async def run(self, path: str, **kwargs):
        defaults = {
            "file_hash": "doc-1",
            "owner_user_id": "user-1",
            "connector_type": "local",
            "file_size": 1234,
        }
        return await self.processor.process_document_standard(
            file_path=path, **{**defaults, **kwargs}
        )


@pytest.fixture
def text_file(tmp_path):
    path = tmp_path / "notes.txt"
    path.write_text("First paragraph.\n\nSecond paragraph.")
    return str(path)


@pytest.fixture
def pdf_file(tmp_path):
    path = tmp_path / "report.pdf"
    path.write_bytes(b"%PDF-fake")
    return str(path)


# --- one trace per file ----------------------------------------------------


@pytest.mark.asyncio
async def test_an_indexed_text_file_is_one_trace_with_its_stages(monkeypatch, finished, text_file):
    pipeline = _Pipeline(monkeypatch)

    with trace_context(session_id="task-1", attributes={"openrag.task_id": "task-1"}):
        result = await pipeline.run(text_file)

    assert result["status"] == "indexed"
    exported = finished()
    root = exported["ingest.file"]
    assert root.parent is None
    assert set(exported) == {
        "ingest.file",
        "ingest.parse",
        "ingest.token_cap",
        "ingest.embed",
        "ingest.index",
    }
    for name in set(exported) - {"ingest.file"}:
        assert exported[name].context.trace_id == root.context.trace_id

    assert root.attributes[LF.TRACE_USER_ID] == "user-1"
    assert root.attributes[LF.TRACE_SESSION_ID] == "task-1"
    assert "ingest_path:openrag" in root.attributes[LF.TRACE_TAGS]
    assert _trace_metadata(root, "task_id") == "task-1"
    assert _trace_metadata(root, "document_id") == "doc-1"
    assert _trace_metadata(root, "ingest_path") == "openrag"
    assert _trace_metadata(root, "connector_type") == "local"
    assert _trace_metadata(root, "embedding_model") == "text-embedding-3-small"
    assert _trace_metadata(root, "embedding_provider") == "openai"
    assert _trace_metadata(root, "run_mode") == "oss"
    assert _trace_metadata(root, "file.extension") == ".txt"
    assert _metadata(root, "status") == "indexed"
    assert _metadata(root, "chunks") == 1


@pytest.mark.asyncio
async def test_stage_counts_for_a_text_file(monkeypatch, finished, text_file):
    pipeline = _Pipeline(monkeypatch)

    await pipeline.run(text_file)

    exported = finished()
    parse = exported["ingest.parse"]
    assert _metadata(parse, "parser") == "Text Parser"
    assert _metadata(parse, "chunks") == 1

    cap = exported["ingest.token_cap"]
    assert _metadata(cap, "max_tokens") == 8000
    assert _metadata(cap, "chunks_in") == 1
    assert _metadata(cap, "chunks_out") == 1

    embed = exported["ingest.embed"]
    assert _metadata(embed, "batches") == 1
    assert _metadata(embed, "inputs") == 1
    assert _metadata(embed, "vectors") == 1
    assert _metadata(embed, "dimensions") == 3
    assert _metadata(embed, "prompt_tokens") == 5

    index = exported["ingest.index"]
    assert _metadata(index, "stale_chunks_deleted") == 2
    assert _metadata(index, "chunks_indexed") == 1


@pytest.mark.asyncio
async def test_embedding_calls_nest_under_the_embed_stage(monkeypatch, finished, text_file):
    pipeline = _Pipeline(monkeypatch)

    await pipeline.run(text_file)

    assert pipeline.embed_parents == ["ingest.embed"]


@pytest.mark.asyncio
async def test_docling_parse_and_extract_counts(monkeypatch, finished, pdf_file):
    pipeline = _Pipeline(monkeypatch)

    await pipeline.run(pdf_file, ocr=None, picture_descriptions=True)

    exported = finished()
    parse = exported["ingest.parse"]
    assert _metadata(parse, "parser").startswith("Docling")
    assert _metadata(parse, "pages") == 2
    assert _metadata(parse, "texts") == 3
    assert _metadata(parse, "tables") == 1
    assert _metadata(parse, "pictures") == 1
    # Unset per-request options fall back to the knowledge config, as Docling does.
    assert _metadata(parse, "ocr") is True
    assert _metadata(parse, "picture_descriptions") is True
    assert _metadata(parse, "table_structure") is True

    extract = exported["ingest.extract"]
    assert _metadata(extract, "chunks") == 3
    assert _metadata(extract, "chunks_text") == 2
    assert _metadata(extract, "chunks_table") == 1
    assert _metadata(extract, "chunks_picture") is None


@pytest.mark.asyncio
async def test_resplit_records_its_window_and_counts(monkeypatch, finished, text_file):
    pipeline = _Pipeline(monkeypatch)

    await pipeline.run(text_file, chunk_size=10, chunk_overlap=2)

    resplit = finished()["ingest.resplit"]
    assert _metadata(resplit, "chunk_size") == 10
    assert _metadata(resplit, "chunk_overlap") == 2
    assert _metadata(resplit, "chunks_in") == 1
    assert _metadata(resplit, "chunks_out") == 5


# --- short and failed traces ------------------------------------------------


@pytest.mark.asyncio
async def test_an_unchanged_file_is_a_short_trace_without_error(monkeypatch, finished, text_file):
    pipeline = _Pipeline(monkeypatch, exists=True)

    result = await pipeline.run(text_file)

    assert result == {"status": "unchanged", "id": "doc-1"}
    exported = finished()
    assert set(exported) == {"ingest.file"}
    root = exported["ingest.file"]
    assert _metadata(root, "status") == "unchanged"
    assert root.attributes.get(LF.OBSERVATION_LEVEL) != "ERROR"


@pytest.mark.asyncio
async def test_an_embedding_failure_marks_the_stage_and_the_trace(monkeypatch, finished, text_file):
    from services.llm_gateway import LlmGatewayError

    pipeline = _Pipeline(monkeypatch)
    pipeline.embed_error = LlmGatewayError("upstream said no", 502)

    with pytest.raises(LlmGatewayError):
        await pipeline.run(text_file)

    exported = finished()
    assert exported["ingest.embed"].attributes[LF.OBSERVATION_LEVEL] == "ERROR"
    root = exported["ingest.file"]
    assert root.attributes[LF.OBSERVATION_LEVEL] == "ERROR"
    assert root.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "LlmGatewayError in ingest.embed"
    assert "ingest.index" not in exported


@pytest.mark.asyncio
async def test_a_document_without_text_marks_the_trace_failed(monkeypatch, finished, tmp_path):
    pipeline = _Pipeline(monkeypatch)
    empty = tmp_path / "empty.txt"
    empty.write_text("   ")

    result = await pipeline.run(str(empty))

    assert result["status"] == "error"
    root = finished()["ingest.file"]
    assert root.attributes[LF.OBSERVATION_LEVEL] == "ERROR"
    assert root.attributes[LF.OBSERVATION_STATUS_MESSAGE] == "NoEmbeddings"


# --- content policy ---------------------------------------------------------


@pytest.mark.asyncio
async def test_filename_is_left_out_unless_capture_is_on(monkeypatch, finished, text_file):
    pipeline = _Pipeline(monkeypatch)

    await pipeline.run(text_file, original_filename="q3-salaries.txt")
    for span in finished().values():
        assert "q3-salaries" not in str(dict(span.attributes))

    monkeypatch.setattr(spans, "content_capture_enabled", lambda: True)
    await pipeline.run(text_file, original_filename="q3-salaries.txt")
    assert _metadata(finished()["ingest.file"], "filename") == "q3-salaries.txt"


@pytest.mark.asyncio
async def test_chunk_text_never_reaches_a_span(monkeypatch, finished, text_file):
    pipeline = _Pipeline(monkeypatch)

    await pipeline.run(text_file)

    for span in finished().values():
        assert "First paragraph" not in str(dict(span.attributes))


@pytest.mark.asyncio
async def test_untraced_ingest_behaves_the_same(monkeypatch, text_file):
    monkeypatch.setattr(spans, "_client", None)
    pipeline = _Pipeline(monkeypatch)

    result = await pipeline.run(text_file)

    assert result == {"status": "indexed", "id": "doc-1", "chunk_count": 1}
    assert len(pipeline.indexed) == 1