Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
5.1 KB
Raw Blame
"""Two files heading for the same indexed filename, ingested at once.

Files inside a task run concurrently under TaskService's worker semaphore, and
the duplicate gate's question — "is this name already indexed?" — is answered
by OpenSearch, which cannot see a file that has not been written yet. Two files
named report.pdf (different folders in an upload, different keys under one
bucket prefix, a Drive selection spanning folders) therefore both passed the
gate, and their chunk ids are content-derived, so neither overwrote the other:
one filename ended up holding two documents' chunks. With replace_duplicates on
— the default for uploads — the later file deleted the earlier one's chunks
instead, while the earlier file's task still reported success.

The in-flight claim makes the second file resolve as an ordinary duplicate.
"""

import asyncio
from unittest.mock import AsyncMock, MagicMock, patch

import pytest

from models.processors import DUPLICATE_FILENAME_WARNING, DocumentFileProcessor
from models.tasks import FileTask, TaskStatus, UploadTask


def _processor(replace_duplicates: bool) -> DocumentFileProcessor:
    processor = DocumentFileProcessor(
        document_service=MagicMock(),
        models_service=MagicMock(),
        owner_user_id="user-123",
        jwt_token="mock-token",
        replace_duplicates=replace_duplicates,
        session_manager=MagicMock(),
        allow_anonymous_delete=True,
    )
    # The index is empty as far as either file can tell: this is the window
    # where both of them are told the name is free.
    processor.check_filename_exists = AsyncMock(return_value=False)
    processor.delete_document_by_filename = AsyncMock(return_value=1)
    processor.process_document_standard = AsyncMock(return_value={"status": "indexed"})
    return processor


async def _ingest_both_at_once(processor, upload_task, file_tasks):
    with (
        patch("os.path.getsize", return_value=1234),
        patch("models.processors.hash_id", side_effect=lambda path: f"hash-{path}"),
    ):
        await asyncio.gather(
            *(
                processor.process_item(upload_task, file_task.file_path, file_task)
                for file_task in file_tasks
            )
        )


@pytest.mark.asyncio
async def test_only_one_of_two_same_named_files_is_ingested():
    processor = _processor(replace_duplicates=False)
    upload_task = UploadTask(task_id="task-123", total_files=2)
    first = FileTask(file_path="/tmp/one/report.pdf", filename="report.pdf")
    second = FileTask(file_path="/tmp/two/report.pdf", filename="report.pdf")

    await _ingest_both_at_once(processor, upload_task, [first, second])

    assert {first.status, second.status} == {TaskStatus.COMPLETED, TaskStatus.SKIPPED}
    # One document indexed, not two under one name.
    processor.process_document_standard.assert_called_once()

    skipped = first if first.status is TaskStatus.SKIPPED else second
    assert skipped.result["reason"] == "duplicate_filename"
    assert skipped.result["warning"] == DUPLICATE_FILENAME_WARNING
    assert skipped.error is None
    # A declined duplicate is a chosen outcome, not a failure.
    assert upload_task.failed_files == 0
    assert upload_task.successful_files == 2


@pytest.mark.asyncio
async def test_replace_duplicates_does_not_let_the_second_file_delete_the_first():
    """With replace on, the loser used to delete the winner's freshly written
    chunks and index itself over them."""
    processor = _processor(replace_duplicates=True)
    upload_task = UploadTask(task_id="task-123", total_files=2)
    first = FileTask(file_path="/tmp/one/report.pdf", filename="report.pdf")
    second = FileTask(file_path="/tmp/two/report.pdf", filename="report.pdf")

    await _ingest_both_at_once(processor, upload_task, [first, second])

    processor.process_document_standard.assert_called_once()
    processor.delete_document_by_filename.assert_not_called()


@pytest.mark.asyncio
async def test_aliases_of_one_name_also_resolve_to_one_file():
    """notes.txt is indexed as notes.md by the Langflow path, so the pair is one
    name for duplicate purposes."""
    processor = _processor(replace_duplicates=False)
    upload_task = UploadTask(task_id="task-123", total_files=2)
    txt = FileTask(file_path="/tmp/notes.txt", filename="notes.txt")
    md = FileTask(file_path="/tmp/notes.md", filename="notes.md")

    await _ingest_both_at_once(processor, upload_task, [txt, md])

    processor.process_document_standard.assert_called_once()
    assert {txt.status, md.status} == {TaskStatus.COMPLETED, TaskStatus.SKIPPED}


@pytest.mark.asyncio
async def test_differently_named_files_are_both_ingested():
    """The claim must not serialize an ordinary batch."""
    processor = _processor(replace_duplicates=False)
    upload_task = UploadTask(task_id="task-123", total_files=2)
    first = FileTask(file_path="/tmp/a.pdf", filename="a.pdf")
    second = FileTask(file_path="/tmp/b.pdf", filename="b.pdf")

    await _ingest_both_at_once(processor, upload_task, [first, second])

    assert first.status is TaskStatus.COMPLETED
    assert second.status is TaskStatus.COMPLETED
    assert processor.process_document_standard.await_count == 2