Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
16.4 KB
Raw Blame
"""Unit tests for the unified duplicate-filename gate across processors.

All processors resolve a filename duplicate the same way via
``TaskProcessor.resolve_duplicate_filename``: with ``replace_duplicates=False``
the file is SKIPPED (counted successful, with a warning); with
``replace_duplicates=True`` the existing document is deleted and ingestion
proceeds.
"""

from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from models.processors import (
    DUPLICATE_FILENAME_WARNING,
    DocumentFileProcessor,
    LangflowFileProcessor,
    S3FileProcessor,
)
from models.tasks import FileTask, TaskStatus, UploadTask

DUPLICATE_SKIPPED_RESULT = {
    "status": "skipped",
    "reason": "duplicate_filename",
    "warning": DUPLICATE_FILENAME_WARNING,
}


def _assert_duplicate_skipped(upload_task: UploadTask, file_task: FileTask) -> None:
    assert file_task.status == TaskStatus.SKIPPED
    assert file_task.error is None
    assert file_task.result == DUPLICATE_SKIPPED_RESULT
    assert upload_task.failed_files == 0
    assert upload_task.successful_files == 1


@pytest.mark.asyncio
async def test_traditional_processor_duplicate_exists_no_replace():
    """A duplicate with replace_duplicates=False is skipped, not failed."""
    mock_doc_service = MagicMock()
    mock_models_service = MagicMock()
    mock_session_manager = MagicMock()

    processor = DocumentFileProcessor(
        document_service=mock_doc_service,
        models_service=mock_models_service,
        owner_user_id="user-123",
        jwt_token="mock-token",
        replace_duplicates=False,
        session_manager=mock_session_manager,
        allow_anonymous_delete=True,
    )

    # Assert that session_manager was set correctly on the processor
    assert processor.session_manager == mock_session_manager

    # Mock base class methods directly on the instance to ensure perfect isolation
    processor.check_filename_exists = AsyncMock(return_value=True)
    processor.delete_document_by_filename = AsyncMock()

    upload_task = UploadTask(task_id="task-123", total_files=1)
    file_task = FileTask(file_path="/tmp/test.txt", filename="test.txt")

    await processor.process_item(upload_task, "/tmp/test.txt", file_task)

    _assert_duplicate_skipped(upload_task, file_task)

    processor.check_filename_exists.assert_called_once()
    processor.delete_document_by_filename.assert_not_called()
    mock_session_manager.get_user_opensearch_client.assert_called_once_with(
        "user-123", "mock-token"
    )


@pytest.mark.asyncio
async def test_traditional_processor_duplicate_exists_with_replace():
    """Verify that if a duplicate file exists and replace_duplicates is True, the old document is deleted and ingestion succeeds."""
    mock_doc_service = MagicMock()
    mock_models_service = MagicMock()
    mock_session_manager = MagicMock()

    processor = DocumentFileProcessor(
        document_service=mock_doc_service,
        models_service=mock_models_service,
        owner_user_id="user-123",
        jwt_token="mock-token",
        replace_duplicates=True,
        session_manager=mock_session_manager,
        allow_anonymous_delete=True,
    )

    # Assert that session_manager was set correctly on the processor
    assert processor.session_manager == mock_session_manager

    # Mock base class methods directly on the instance to ensure perfect isolation
    processor.check_filename_exists = AsyncMock(return_value=True)
    processor.delete_document_by_filename = AsyncMock(return_value=1)
    processor.process_document_standard = AsyncMock(return_value={"status": "indexed"})

    upload_task = UploadTask(task_id="task-123", total_files=1)
    file_task = FileTask(file_path="/tmp/test.txt", filename="test.txt")

    with (
        patch("os.path.getsize", return_value=1234),
        patch("models.processors.hash_id", return_value="dummy-hash"),
    ):
        await processor.process_item(upload_task, "/tmp/test.txt", file_task)

    assert file_task.status == TaskStatus.COMPLETED
    assert file_task.error is None
    assert upload_task.failed_files == 0
    assert upload_task.successful_files == 1

    processor.check_filename_exists.assert_called_once()
    processor.delete_document_by_filename.assert_called_once()
    # The delete must be owner-scoped so it actually removes the old chunks.
    delete_call = processor.delete_document_by_filename.call_args
    assert delete_call.kwargs["owner_user_id"] == "user-123"
    processor.process_document_standard.assert_called_once()
    mock_session_manager.get_user_opensearch_client.assert_called_once_with(
        "user-123", "mock-token"
    )


@pytest.mark.asyncio
async def test_langflow_processor_duplicate_exists_no_replace():
    """LangflowFileProcessor skips duplicates with the same uniform payload."""
    mock_session_manager = MagicMock()
    langflow_file_service = MagicMock()
    langflow_file_service.upload_and_ingest_file = AsyncMock()

    processor = LangflowFileProcessor(
        langflow_file_service=langflow_file_service,
        session_manager=mock_session_manager,
        owner_user_id="user-123",
        jwt_token="mock-token",
        replace_duplicates=False,
        allow_anonymous_delete=True,
    )
    processor.check_filename_exists = AsyncMock(return_value=True)
    processor.delete_document_by_filename = AsyncMock()

    upload_task = UploadTask(task_id="task-123", total_files=1)
    file_task = FileTask(file_path="/tmp/test.txt", filename="test.txt")

    await processor.process_item(upload_task, "/tmp/test.txt", file_task)

    _assert_duplicate_skipped(upload_task, file_task)
    processor.delete_document_by_filename.assert_not_called()
    langflow_file_service.upload_and_ingest_file.assert_not_called()


@pytest.mark.asyncio
async def test_resolve_duplicate_skips_when_delete_removes_nothing():
    """If replace is requested but deletion removes 0 chunks (e.g. owner_user_id
    missing on a private sync), resolve_duplicate_filename must return 'skip'
    instead of falsely reporting 'replaced'."""
    from models.processors import TaskProcessor

    processor = TaskProcessor()
    opensearch_client = AsyncMock()

    processor.check_filename_exists = AsyncMock(return_value=True)
    processor.delete_document_by_filename = AsyncMock(return_value=0)

    result = await processor.resolve_duplicate_filename(
        "report.pdf",
        opensearch_client,
        replace=True,
        owner_user_id=None,
        allow_anonymous_delete=True,
    )

    assert result == "skip"
    processor.delete_document_by_filename.assert_awaited_once()


@pytest.mark.asyncio
async def test_delete_document_by_filename_shared_without_owner(monkeypatch):
    """shared=True with no owner_user_id must use the anonymous filename query
    instead of silently skipping deletion."""
    from types import SimpleNamespace

    from models.processors import TaskProcessor

    admin_client = AsyncMock()
    admin_client.delete = AsyncMock(return_value={"result": "deleted"})
    monkeypatch.setattr(
        "config.settings.clients",
        SimpleNamespace(opensearch=admin_client),
    )
    monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")

    opensearch_client = AsyncMock()
    opensearch_client.search = AsyncMock(
        return_value={"_scroll_id": None, "hits": {"hits": [{"_id": "chunk-1"}]}}
    )

    processor = TaskProcessor()
    deleted = await processor.delete_document_by_filename(
        "report.pdf",
        opensearch_client,
        owner_user_id=None,
        shared=True,
        allow_anonymous_delete=True,
    )

    assert deleted == 1
    search_body = opensearch_client.search.await_args.kwargs["body"]
    filters = search_body["query"]["bool"]["filter"]
    assert {"bool": {"must_not": {"exists": {"field": "owner"}}}} in filters


@pytest.mark.asyncio
async def test_delete_document_by_filename_replaces_owned_and_ownerless(monkeypatch):
    """With an owner in hand, and the permission to delete shared documents,
    the replace scope is "mine OR ownerless".

    ``shared`` says how the replacement will be written; it must not narrow what
    gets deleted. An owner-only scope matched none of a shared document's
    chunks, so the caller saw zero deletions, reported "nothing to replace" and
    skipped the file — even though the (owner-agnostic) duplicate check had just
    found it and the user had confirmed the overwrite.
    """
    from types import SimpleNamespace

    from models.processors import TaskProcessor

    admin_client = AsyncMock()
    admin_client.delete = AsyncMock(return_value={"result": "deleted"})
    monkeypatch.setattr(
        "config.settings.clients",
        SimpleNamespace(opensearch=admin_client),
    )
    monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")

    opensearch_client = AsyncMock()
    opensearch_client.search = AsyncMock(
        return_value={"_scroll_id": None, "hits": {"hits": [{"_id": "chunk-1"}]}}
    )

    processor = TaskProcessor()
    deleted = await processor.delete_document_by_filename(
        "report.pdf",
        opensearch_client,
        owner_user_id="user-123",
        shared=False,
        allow_anonymous_delete=True,
    )

    assert deleted == 1
    filters = opensearch_client.search.await_args.kwargs["body"]["query"]["bool"]["filter"]
    assert {
        "bool": {
            "should": [
                {"term": {"owner": "user-123"}},
                {"bool": {"must_not": {"exists": {"field": "owner"}}}},
            ],
            "minimum_should_match": 1,
        }
    } in filters


def _build_s3_processor(replace_duplicates: bool) -> S3FileProcessor:
    document_service = MagicMock()
    document_service.session_manager = MagicMock()
    processor = S3FileProcessor(
        document_service,
        bucket="test-bucket",
        s3_client=MagicMock(),
        owner_user_id="user-123",
        jwt_token="mock-token",
        models_service=MagicMock(),
        docling_service=MagicMock(),
        replace_duplicates=replace_duplicates,
        allow_anonymous_delete=True,
    )
    return processor


@pytest.mark.asyncio
async def test_s3_processor_duplicate_exists_no_replace():
    """S3FileProcessor now has the same duplicate gate: same-key duplicates
    are skipped before the object is even downloaded."""
    processor = _build_s3_processor(replace_duplicates=False)
    processor.check_filename_exists = AsyncMock(return_value=True)
    processor.delete_document_by_filename = AsyncMock()

    upload_task = UploadTask(task_id="task-123", total_files=1)
    file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")

    await processor.process_item(upload_task, "docs/report.pdf", file_task)

    _assert_duplicate_skipped(upload_task, file_task)
    processor.delete_document_by_filename.assert_not_called()
    processor.s3_client.download_fileobj.assert_not_called()


@pytest.mark.asyncio
async def test_s3_processor_duplicate_exists_with_replace(tmp_path):
    """With replace_duplicates=True the old document is deleted and the object
    is downloaded and processed."""
    processor = _build_s3_processor(replace_duplicates=True)
    processor.check_filename_exists = AsyncMock(return_value=True)
    processor.delete_document_by_filename = AsyncMock(return_value=1)
    processor.process_document_standard = AsyncMock(
        return_value={"status": "indexed", "id": "hash-1"}
    )
    processor.s3_client.head_object = MagicMock(return_value={"ContentLength": 10})

    upload_task = UploadTask(task_id="task-123", total_files=1)
    file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")

    with patch("models.processors.hash_id", return_value="dummy-hash"):
        await processor.process_item(upload_task, "docs/report.pdf", file_task)

    assert file_task.status == TaskStatus.COMPLETED
    assert upload_task.successful_files == 1
    processor.delete_document_by_filename.assert_called_once()
    assert processor.delete_document_by_filename.call_args.kwargs["owner_user_id"] == "user-123"
    processor.s3_client.download_fileobj.assert_called_once()
    processor.process_document_standard.assert_awaited_once()


@pytest.mark.asyncio
async def test_s3_processor_no_duplicate_proceeds():
    """No duplicate: the gate is a no-op and processing continues."""
    processor = _build_s3_processor(replace_duplicates=False)
    processor.check_filename_exists = AsyncMock(return_value=False)
    processor.delete_document_by_filename = AsyncMock()
    processor.process_document_standard = AsyncMock(
        return_value={"status": "indexed", "id": "hash-1"}
    )
    processor.s3_client.head_object = MagicMock(return_value={"ContentLength": 10})

    upload_task = UploadTask(task_id="task-123", total_files=1)
    file_task = FileTask(file_path="docs/report.pdf", filename="docs/report.pdf")

    with patch("models.processors.hash_id", return_value="dummy-hash"):
        await processor.process_item(upload_task, "docs/report.pdf", file_task)

    assert file_task.status == TaskStatus.COMPLETED
    processor.delete_document_by_filename.assert_not_called()
    processor.process_document_standard.assert_awaited_once()


def _delete_scope_env(monkeypatch):
    """Admin write client + a user client that reports one visible chunk."""
    from types import SimpleNamespace

    admin_client = AsyncMock()
    admin_client.delete = AsyncMock(return_value={"result": "deleted"})
    monkeypatch.setattr(
        "config.settings.clients",
        SimpleNamespace(opensearch=admin_client),
    )
    monkeypatch.setattr("config.settings.get_index_name", lambda: "test-index")

    opensearch_client = AsyncMock()
    opensearch_client.search = AsyncMock(
        return_value={"_scroll_id": None, "hits": {"hits": [{"_id": "chunk-1"}]}}
    )
    return opensearch_client


@pytest.mark.asyncio
async def test_replace_without_anonymous_delete_permission_stays_owner_scoped(monkeypatch):
    """Ownerless chunks are visible to the whole instance, so replacing a
    document that turns out to be one is a deletion of shared content. Without
    knowledge:delete:anonymous the scope must stay owner-only, leaving someone
    else's shared document alone."""
    from models.processors import TaskProcessor

    opensearch_client = _delete_scope_env(monkeypatch)

    await TaskProcessor().delete_document_by_filename(
        "report.pdf",
        opensearch_client,
        owner_user_id="user-123",
        shared=False,
        allow_anonymous_delete=False,
    )

    from utils.opensearch_queries import build_owned_filename_query

    query = opensearch_client.search.await_args.kwargs["body"]["query"]
    assert query == build_owned_filename_query("report.pdf", "user-123")


@pytest.mark.asyncio
async def test_shared_does_not_stand_in_for_the_permission(monkeypatch):
    """``shared`` says how the replacement is written, and on the re-sync paths
    it is inferred from what is indexed. Letting it widen the delete meant a
    caller without knowledge:delete:anonymous could replace a shared document
    by re-syncing it. An explicit shared sync loses nothing by this: it cannot
    get here without the permission (connector_sync returns 403)."""
    from models.processors import TaskProcessor

    opensearch_client = _delete_scope_env(monkeypatch)

    await TaskProcessor().delete_document_by_filename(
        "report.pdf",
        opensearch_client,
        owner_user_id="user-123",
        shared=True,
        allow_anonymous_delete=False,
    )

    from utils.opensearch_queries import build_owned_filename_query

    query = opensearch_client.search.await_args.kwargs["body"]["query"]
    assert query == build_owned_filename_query("report.pdf", "user-123")


@pytest.mark.asyncio
async def test_resolve_duplicate_filename_forwards_the_permission(monkeypatch):
    """The gate every processor shares hands its caller's resolved permission to
    the delete, rather than deciding the scope on its own."""
    from models.processors import TaskProcessor

    processor = TaskProcessor()
    processor.check_filename_exists = AsyncMock(return_value=True)
    processor.delete_document_by_filename = AsyncMock(return_value=1)

    from types import SimpleNamespace

    monkeypatch.setattr(
        "models.processors.clients",
        SimpleNamespace(opensearch=AsyncMock()),
    )

    await processor.resolve_duplicate_filename(
        "report.pdf",
        AsyncMock(),
        replace=True,
        owner_user_id="user-123",
        allow_anonymous_delete=False,
    )

    assert (
        processor.delete_document_by_filename.await_args.kwargs["allow_anonymous_delete"] is False
    )