Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
16.6 KB
Raw Blame
"""Pruning TTL validation, prune rollback races, and file-save failure handling."""

import asyncio
import errno
import json
import sys
from datetime import UTC, datetime, timedelta
from pathlib import Path

import pytest
import pytest_asyncio
from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine
from sqlmodel import SQLModel

ROOT = Path(__file__).resolve().parent.parent.parent.parent
SRC = ROOT / "src"
if str(SRC) not in sys.path:
    sys.path.insert(0, str(SRC))

import db.models  # noqa: E402,F401
from db.repositories import ConversationRepo  # noqa: E402
from services.conversation_persistence_service import (  # noqa: E402
    ConversationPersistenceService,
)


@pytest_asyncio.fixture
async def session_factory():
    engine = create_async_engine(
        "sqlite+aiosqlite:///:memory:",
        connect_args={"check_same_thread": False},
    )
    async with engine.begin() as conn:
        await conn.run_sync(SQLModel.metadata.create_all)
    factory = async_sessionmaker(engine, expire_on_commit=False)
    yield factory
    await engine.dispose()


@pytest.fixture
def storage_path(tmp_path):
    return tmp_path / "conversations.json"


def _svc(storage_path, session_factory) -> ConversationPersistenceService:
    return ConversationPersistenceService(
        storage_file=str(storage_path),
        session_factory=session_factory,
    )


def _payload(response_id="r-1", title="Hello", *, age_days=0) -> dict:
    last_activity = datetime.now() - timedelta(days=age_days)
    return {
        "response_id": response_id,
        "title": title,
        "endpoint": "chat",
        "previous_response_id": None,
        "filter_id": None,
        "total_messages": 1,
        "last_activity": last_activity.isoformat(),
    }


async def _seed_db(session_factory, response_id: str, *, user_id="alice", age_days=0) -> None:
    async with session_factory() as session:
        await ConversationRepo(session).upsert(
            response_id=response_id,
            user_id=user_id,
            title="db row",
            last_activity=datetime.now(UTC) - timedelta(days=age_days),
        )
        await session.commit()


async def _db_ids(session_factory) -> set[str]:
    async with session_factory() as session:
        rows = await ConversationRepo(session).list_for_user("alice")
    return {row.response_id for row in rows}


async def _failing_save() -> None:
    raise OSError("disk full")


# ---------------------------------------------------------------------------
# 1. ttl_days must be >= 1
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
@pytest.mark.parametrize("ttl_days", [0, -1, -30])
async def test_service_rejects_non_positive_ttl_and_changes_nothing(
    monkeypatch, storage_path, session_factory, ttl_days
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "hybrid")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "old", _payload("old", age_days=90))
    await _seed_db(session_factory, "old-db", age_days=90)
    on_disk_before = storage_path.read_text()
    memory_before = json.loads(json.dumps(svc._conversations))

    with pytest.raises(ValueError, match="at least 1"):
        await svc.prune_stale_conversations(ttl_days)

    assert svc._conversations == memory_before
    assert storage_path.read_text() == on_disk_before
    assert "old-db" in await _db_ids(session_factory)


@pytest.mark.asyncio
@pytest.mark.parametrize("ttl_days", [0, -1, -30])
async def test_repo_rejects_non_positive_ttl_and_deletes_nothing(session_factory, ttl_days):
    await _seed_db(session_factory, "old-db", age_days=90)

    async with session_factory() as session:
        with pytest.raises(ValueError, match="at least 1"):
            await ConversationRepo(session).delete_stale(ttl_days)

    assert await _db_ids(session_factory) == {"old-db"}


@pytest.mark.asyncio
async def test_positive_ttl_prunes_old_and_keeps_recent(monkeypatch, storage_path, session_factory):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "hybrid")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "old", _payload("old", age_days=90))
    await svc.store_conversation_thread("alice", "recent", _payload("recent", age_days=1))
    await _seed_db(session_factory, "old-db", age_days=90)
    await _seed_db(session_factory, "recent-db", age_days=1)

    deleted = await svc.prune_stale_conversations(30)

    assert deleted >= 2
    assert set(svc._conversations["alice"]) == {"recent"}
    assert set(json.loads(storage_path.read_text())["alice"]) == {"recent"}
    ids = await _db_ids(session_factory)
    assert "old-db" not in ids
    assert "recent-db" in ids


@pytest.mark.asyncio
@pytest.mark.parametrize("mode", ["db", "hybrid"])
async def test_prune_propagates_db_failure(monkeypatch, storage_path, session_factory, mode):
    """A DB prune error must reach the caller, not be swallowed after logging."""
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", mode)
    svc = _svc(storage_path, session_factory)
    await _seed_db(session_factory, "old-db", age_days=90)

    async def _boom(self, ttl_days):
        raise RuntimeError("db down")

    monkeypatch.setattr(ConversationRepo, "delete_stale", _boom)

    with pytest.raises(RuntimeError, match="db down"):
        await svc.prune_stale_conversations(30)

    assert "old-db" in await _db_ids(session_factory)


# ---------------------------------------------------------------------------
# 2. Failed-prune rollback must not clobber concurrent updates
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_failed_prune_rollback_keeps_a_newer_concurrent_version(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "files")
    svc = _svc(storage_path, session_factory)
    stale = _payload("old", title="stale version", age_days=90)
    svc._conversations = {"alice": {"old": stale}}

    save_started = asyncio.Event()
    release_save = asyncio.Event()
    calls = 0

    async def save():
        nonlocal calls
        calls += 1
        if calls == 1:  # the prune's own save: pause, then fail
            save_started.set()
            await release_save.wait()
            raise OSError("disk full")

    svc._save_conversations = save

    prune = asyncio.create_task(svc.prune_stale_conversations(30))
    await save_started.wait()
    # Pruning has removed the entry and is blocked in its save; a newer
    # version of the same conversation is stored meanwhile.
    assert "old" not in svc._conversations["alice"]
    newer = _payload("old", title="newer version", age_days=0)
    await svc.store_conversation_thread("alice", "old", newer)
    release_save.set()

    assert await prune == 0
    assert svc._conversations["alice"]["old"]["title"] == "newer version"


@pytest.mark.asyncio
async def test_failed_prune_rollback_restores_an_unchanged_entry(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "files")
    svc = _svc(storage_path, session_factory)
    stale = _payload("old", title="stale version", age_days=90)
    svc._conversations = {"alice": {"old": stale}}
    svc._save_conversations = _failing_save

    assert await svc.prune_stale_conversations(30) == 0

    assert svc._conversations["alice"]["old"] == stale


# ---------------------------------------------------------------------------
# 3. File-save failures must not skip the DB delete
# ---------------------------------------------------------------------------


@pytest.mark.asyncio
async def test_delete_thread_still_deletes_from_db_when_file_save_fails(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "hybrid")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "r-1", _payload("r-1"))
    assert "r-1" in await _db_ids(session_factory)
    svc._save_conversations = _failing_save

    with pytest.raises(OSError, match="disk full"):
        await svc.delete_conversation_thread("alice", "r-1")

    assert "r-1" not in await _db_ids(session_factory)
    assert "r-1" not in svc._conversations.get("alice", {})


@pytest.mark.asyncio
async def test_clear_user_still_deletes_from_db_when_file_save_fails(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "hybrid")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "r-1", _payload("r-1"))
    await svc.store_conversation_thread("alice", "r-2", _payload("r-2"))
    assert await _db_ids(session_factory) == {"r-1", "r-2"}
    svc._save_conversations = _failing_save

    with pytest.raises(OSError, match="disk full"):
        await svc.clear_user_conversations("alice")

    assert await _db_ids(session_factory) == set()
    assert "alice" not in svc._conversations


@pytest.mark.asyncio
async def test_files_only_delete_failure_is_logged_not_durable(
    monkeypatch, storage_path, session_factory
):
    """In files mode a failed save leaves the entry on disk.

    The error is raised so the caller knows the delete was not durable: a
    restart before another successful save brings the conversation back. A
    later successful save rewrites the file from memory.
    """
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "files")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "r-1", _payload("r-1"))
    await svc.store_conversation_thread("alice", "r-2", _payload("r-2"))
    real_save = svc._save_conversations
    svc._save_conversations = _failing_save

    with pytest.raises(OSError, match="disk full"):
        await svc.delete_conversation_thread("alice", "r-1")

    assert "r-1" not in svc._conversations["alice"]
    assert set(json.loads(storage_path.read_text())["alice"]) == {"r-1", "r-2"}
    reloaded = _svc(storage_path, session_factory)
    assert "r-1" in reloaded._conversations["alice"]

    svc._save_conversations = real_save
    await svc._save_conversations()
    assert set(json.loads(storage_path.read_text())["alice"]) == {"r-2"}


@pytest.mark.asyncio
async def test_files_only_clear_failure_is_logged_not_durable(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "files")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "r-1", _payload("r-1"))
    svc._save_conversations = _failing_save

    with pytest.raises(OSError, match="disk full"):
        await svc.clear_user_conversations("alice")

    assert "alice" not in svc._conversations
    assert "r-1" in json.loads(storage_path.read_text())["alice"]


@pytest.mark.asyncio
async def test_files_only_delete_retry_saves_again_after_failed_save(
    monkeypatch, storage_path, session_factory
):
    """A retry must rewrite the file even though memory no longer has the entry."""
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "files")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "r-1", _payload("r-1"))
    await svc.store_conversation_thread("alice", "r-2", _payload("r-2"))
    real_save = svc._save_conversations
    svc._save_conversations = _failing_save

    with pytest.raises(OSError, match="disk full"):
        await svc.delete_conversation_thread("alice", "r-1")

    svc._save_conversations = real_save
    assert await svc.delete_conversation_thread("alice", "r-1") is True
    assert set(json.loads(storage_path.read_text())["alice"]) == {"r-2"}

    # Once the file reflects the delete, nothing is pending any more.
    assert await svc.delete_conversation_thread("alice", "r-1") is False


@pytest.mark.asyncio
async def test_files_only_clear_retry_saves_again_after_failed_save(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "files")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "r-1", _payload("r-1"))
    real_save = svc._save_conversations
    svc._save_conversations = _failing_save

    with pytest.raises(OSError, match="disk full"):
        await svc.clear_user_conversations("alice")

    svc._save_conversations = real_save
    await svc.clear_user_conversations("alice")
    assert "alice" not in json.loads(storage_path.read_text())


# ---------------------------------------------------------------------------
# 4. A failed prune must not resurrect a conversation deleted meanwhile
# ---------------------------------------------------------------------------


async def _pause_then_fail_first_save(svc):
    """Make the prune's save block until released, then fail; later saves succeed."""
    save_started = asyncio.Event()
    release_save = asyncio.Event()
    real_save = svc._save_conversations
    calls = 0

    async def save():
        nonlocal calls
        calls += 1
        if calls == 1:
            save_started.set()
            await release_save.wait()
            raise OSError("disk full")
        await real_save()

    svc._save_conversations = save
    return save_started, release_save


@pytest.mark.asyncio
async def test_failed_prune_does_not_resurrect_an_explicitly_deleted_conversation(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "hybrid")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "old", _payload("old", age_days=90))
    save_started, release_save = await _pause_then_fail_first_save(svc)

    prune = asyncio.create_task(svc.prune_stale_conversations(30))
    await save_started.wait()
    # The prune has removed "old" from memory and is stuck in its save. An
    # explicit delete now finds nothing to remove from the file store.
    delete = asyncio.create_task(svc.delete_conversation_thread("alice", "old"))
    await asyncio.sleep(0)  # let the delete run as far as it can
    release_save.set()
    await prune
    assert await delete is True

    assert "old" not in svc._conversations.get("alice", {})
    assert "old" not in await svc.get_user_conversations("alice")
    assert "old" not in await _db_ids(session_factory)


@pytest.mark.asyncio
async def test_failed_prune_does_not_resurrect_a_cleared_user(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "hybrid")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "old", _payload("old", age_days=90))
    save_started, release_save = await _pause_then_fail_first_save(svc)

    prune = asyncio.create_task(svc.prune_stale_conversations(30))
    await save_started.wait()
    clear = asyncio.create_task(svc.clear_user_conversations("alice"))
    await asyncio.sleep(0)
    release_save.set()
    await prune
    await clear

    assert not svc._conversations.get("alice")
    assert await svc.get_user_conversations("alice") == {}
    assert await _db_ids(session_factory) == set()


# ---------------------------------------------------------------------------
# 5. Directory fsync after the atomic replace
# ---------------------------------------------------------------------------


def _fail_dir_fsync(monkeypatch, err):
    import services.conversation_persistence_service as mod

    def boom(path):
        raise err

    monkeypatch.setattr(mod.ConversationPersistenceService, "_fsync_dir", staticmethod(boom))


@pytest.mark.parametrize("code", [errno.EINVAL, errno.ENOTSUP, errno.EOPNOTSUPP])
def test_fsync_dir_ignores_unsupported_errors(monkeypatch, tmp_path, code):
    import services.conversation_persistence_service as mod

    def unsupported(fd):
        raise OSError(code, "unsupported")

    monkeypatch.setattr(mod.os, "fsync", unsupported)
    ConversationPersistenceService._fsync_dir(str(tmp_path))  # must not raise


def test_fsync_dir_propagates_real_errors(monkeypatch, tmp_path):
    import services.conversation_persistence_service as mod

    def eio(fd):
        raise OSError(errno.EIO, "io error")

    monkeypatch.setattr(mod.os, "fsync", eio)
    with pytest.raises(OSError):
        ConversationPersistenceService._fsync_dir(str(tmp_path))


@pytest.mark.asyncio
async def test_prune_keeps_memory_matching_replaced_file_when_dir_fsync_fails(
    monkeypatch, storage_path, session_factory
):
    monkeypatch.setenv("OPENRAG_STORAGE_MODE", "files")
    svc = _svc(storage_path, session_factory)
    await svc.store_conversation_thread("alice", "old", _payload("old", age_days=90))
    await svc.store_conversation_thread("alice", "recent", _payload("recent"))
    _fail_dir_fsync(monkeypatch, OSError(errno.EIO, "io error"))

    assert await svc.prune_stale_conversations(30) == 1

    assert set(svc._conversations["alice"]) == {"recent"}
    assert set(json.loads(storage_path.read_text())["alice"]) == {"recent"}