Skip to content

OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.

Python
23.6 KB
Raw Blame
"""Conversation persistence — chat-history metadata only (full message
bodies live in Langflow).

Mode-aware (``OPENRAG_STORAGE_MODE`` from ``src/config/storage_mode.py``):

| Mode         | Reads                | Writes              |
|--------------|----------------------|---------------------|
| db (default) | DB only              | DB only — no JSON   |
| hybrid       | DB → JSON fallback   | DB + JSON dual      |
| files        | JSON only            | JSON only           |

All public methods are async. Call sites in ``src/agent.py`` were
flipped from sync to ``await`` as part of this migration.
"""

from __future__ import annotations

import asyncio
import errno
import json
import os
import tempfile
import threading
from collections.abc import Callable
from datetime import datetime, timedelta
from typing import Any

from config.paths import get_data_file
from config.storage_mode import (
    db_writes_enabled,
    file_writes_enabled,
    get_storage_mode,
)
from utils.logging_config import get_logger

logger = get_logger(__name__)


class PostReplaceSyncError(OSError):
    """The store file was replaced, but the directory fsync then failed.

    The new contents are in place; only their durability across a crash is
    unknown. Callers must not roll back in-memory state for this error.
    """


class ConversationPersistenceService:
    """Per-user chat-history index. Stores metadata only."""

    def __init__(
        self,
        storage_file: str | None = None,
        session_factory: Callable | None = None,
    ):
        self.storage_file = storage_file or get_data_file("conversations.json")
        os.makedirs(os.path.dirname(self.storage_file), exist_ok=True)
        self.lock = threading.Lock()
        # Serialises whole saves (snapshot -> write -> replace) so an older
        # snapshot can never replace a newer one. Kept separate from ``lock``
        # so mutators are not blocked during file I/O.
        self._save_lock = threading.Lock()
        # Serialises the JSON-store mutations that prune can roll back (prune,
        # explicit delete, clear), so a failed prune can never resurrect an
        # entry that was deleted while its save was in flight.
        self._file_mutation_lock = asyncio.Lock()
        # Deletions applied in memory whose file save has not yet succeeded.
        # While non-empty, memory and disk may differ, so a retry of the same
        # delete/clear must save again instead of finding nothing to do.
        # Guarded by ``lock``; cleared by any successful save (its snapshot
        # already excludes every removal recorded here).
        self._pending_file_deletes: set[tuple[str, str]] = set()
        self._pending_file_clears: set[str] = set()
        self._session_factory = session_factory
        self._conversations: dict[str, dict[str, Any]] = self._load_conversations()

    # ------------------------------------------------------------------
    # JSON helpers
    # ------------------------------------------------------------------

    def _load_conversations(self) -> dict[str, dict[str, Any]]:
        if os.path.exists(self.storage_file):
            try:
                with open(self.storage_file, encoding="utf-8") as f:
                    return json.load(f)
            except Exception as exc:  # noqa: BLE001
                logger.error(f"Error loading conversations: {exc}")
                return {}
        return {}

    def _save_conversations_sync(self) -> None:
        """Write to a temp file then atomically replace storage_file.

        Takes a shallow snapshot of ``_conversations`` under ``self.lock``
        so the lock is not held during the (potentially slow) file I/O.
        This ensures a failed write (disk full, serialisation error, etc.)
        leaves the existing store intact rather than truncating it.
        """
        with self._save_lock:
            with self.lock:
                snapshot = {uid: dict(convs) for uid, convs in self._conversations.items()}

            dir_ = os.path.dirname(self.storage_file)
            fd, tmp_path = tempfile.mkstemp(dir=dir_, suffix=".tmp")
            try:
                with os.fdopen(fd, "w", encoding="utf-8") as f:
                    json.dump(snapshot, f, indent=2, ensure_ascii=False, default=str)
                    f.flush()
                    os.fsync(f.fileno())
                os.replace(tmp_path, self.storage_file)
            except Exception:
                try:
                    os.unlink(tmp_path)
                except OSError:
                    pass
                raise
            try:
                self._fsync_dir(dir_)
            except OSError as exc:
                raise PostReplaceSyncError(exc.errno, str(exc)) from exc

    @staticmethod
    def _fsync_dir(path: str) -> None:
        """Make the rename durable. Only "unsupported" errors are ignored."""
        unsupported = {errno.EINVAL, errno.ENOTSUP, errno.EOPNOTSUPP}
        try:
            dir_fd = os.open(path, os.O_RDONLY)
        except OSError as exc:
            # Platforms that cannot open a directory (e.g. Windows) cannot fsync it.
            if exc.errno in unsupported or isinstance(exc, PermissionError):
                return
            raise
        try:
            os.fsync(dir_fd)
        except OSError as exc:
            if exc.errno not in unsupported:
                raise
        finally:
            os.close(dir_fd)

    async def _save_conversations(self) -> None:
        loop = asyncio.get_event_loop()
        await loop.run_in_executor(None, self._save_conversations_sync)

    async def _save_after_removal(self) -> None:
        """Save, and forget pending deletions once the file reflects them."""
        await self._save_conversations()
        with self.lock:
            self._pending_file_deletes.clear()
            self._pending_file_clears.clear()

    def _serialize_datetime(self, obj: Any) -> Any:
        if isinstance(obj, datetime):
            return obj.isoformat()
        if isinstance(obj, dict):
            return {k: self._serialize_datetime(v) for k, v in obj.items()}
        if isinstance(obj, list):
            return [self._serialize_datetime(x) for x in obj]
        return obj

    def _count_total(self, data: dict[str, Any]) -> int:
        total = 0
        for user_conv in data.values():
            if isinstance(user_conv, dict):
                total += len(user_conv)
        return total

    # ------------------------------------------------------------------
    # Public async API
    # ------------------------------------------------------------------

    async def get_user_conversations(self, user_id: str) -> dict[str, Any]:
        mode = get_storage_mode()
        if mode == "files":
            if user_id not in self._conversations:
                self._conversations[user_id] = {}
            return self._conversations[user_id]

        # db / hybrid — DB read first
        db_payload = await self._db_get_for_user(user_id)
        if mode == "db":
            return db_payload

        # hybrid — merge JSON entries that aren't yet in DB
        merged = dict(db_payload)
        for resp_id, payload in self._conversations.get(user_id, {}).items():
            merged.setdefault(resp_id, payload)
        return merged

    async def store_conversation_thread(
        self,
        user_id: str,
        response_id: str,
        conversation_state: dict[str, Any],
    ) -> None:
        serialized = self._serialize_datetime(conversation_state)

        if file_writes_enabled():
            with self.lock:
                if user_id not in self._conversations:
                    self._conversations[user_id] = {}
                self._conversations[user_id][response_id] = serialized
            try:
                await self._save_conversations()  # snapshots under lock internally
            except Exception as exc:  # noqa: BLE001
                logger.error(
                    "Failed to persist conversation to disk",
                    user_id=user_id,
                    response_id=response_id,
                    error=str(exc),
                )
                # Save failure is observable via log; do not block the DB write.

        if db_writes_enabled():
            await self._db_upsert(user_id, response_id, serialized)

    async def get_conversation_thread(self, user_id: str, response_id: str) -> dict[str, Any]:
        mode = get_storage_mode()
        if mode != "files":
            payload = await self._db_get_one(response_id, user_id)
            if payload is not None:
                return payload
            if mode == "db":
                return {}
        # files or hybrid-with-no-db-row → JSON
        return self._conversations.get(user_id, {}).get(response_id, {})

    async def delete_conversation_thread(self, user_id: str, response_id: str) -> bool:
        """Delete one conversation from every active store.

        A file-save failure does not skip the DB delete, but it is not hidden:
        the DB delete runs first, then the save error is re-raised so the
        caller knows the JSON copy may survive a restart (and, in hybrid mode,
        be served by the JSON fallback). The entry is already gone from memory,
        so the deletion is recorded as pending: retrying the same delete saves
        the file again and reports the conversation as deleted once it succeeds.
        """
        deleted = False
        save_error: Exception | None = None

        if file_writes_enabled():
            # Waits for an in-flight prune, so its rollback cannot undo this.
            async with self._file_mutation_lock:
                with self.lock:
                    user_convs = self._conversations.get(user_id)
                    removed = (
                        user_convs is not None and user_convs.pop(response_id, None) is not None
                    )
                    if removed:
                        self._pending_file_deletes.add((user_id, response_id))
                    needs_save = (
                        removed
                        or (user_id, response_id) in self._pending_file_deletes
                        or user_id in self._pending_file_clears
                    )
                if needs_save:
                    deleted = True
                    try:
                        await self._save_after_removal()
                    except Exception as exc:  # noqa: BLE001
                        save_error = exc
                        logger.error(
                            "Failed to persist conversation deletion to disk",
                            user_id=user_id,
                            response_id=response_id,
                            error=str(exc),
                        )

        if db_writes_enabled():
            db_deleted = await self._db_delete(response_id, user_id)
            deleted = deleted or db_deleted

        if save_error is not None:
            raise save_error
        return deleted

    async def prune_stale_conversations(self, ttl_days: int) -> int:
        """Delete conversations that have had no activity for *ttl_days* days.

        Covers all active storage modes:
        - ``db`` / ``hybrid``: bulk DELETE via the DB repo.
        - ``files`` / ``hybrid``: removes matching entries from the in-memory
          JSON store and persists to disk.

        The JSON-store part is serialised with ``delete_conversation_thread`` and
        ``clear_user_conversations``: if its save fails and entries are rolled
        back, a delete issued meanwhile waits and then removes the restored
        entry, so a deleted conversation is never resurrected.

        A database failure is logged and then re-raised, so callers can tell an
        incomplete prune from a completed one and retry. In hybrid mode the JSON
        part has already been applied by then; pruning is idempotent, so a retry
        is safe.

        Returns the total number of conversations deleted across all stores.
        Raises ``ValueError`` unless *ttl_days* >= 1, before touching memory,
        disk or the DB: zero would delete nearly every conversation and a
        negative value would move the cutoff into the future.
        """
        if ttl_days < 1:
            raise ValueError(f"ttl_days must be at least 1, got {ttl_days}")

        deleted = 0
        # Use local naive time to match the naive timestamps produced by
        # datetime.now() in agent.py.  Each aware timestamp is individually
        # converted to local naive time via astimezone() so Python picks the
        # correct local UTC offset for that instant rather than relying on a
        # single cached tzinfo object.
        cutoff_local = datetime.now() - timedelta(days=ttl_days)

        if file_writes_enabled():
            async with self._file_mutation_lock:
                deleted += await self._prune_file_store(cutoff_local)

        if db_writes_enabled():
            from db.repositories import ConversationRepo

            sess_factory = self._resolve_session_factory()
            if sess_factory is not None:
                try:
                    async with sess_factory() as session:
                        n = await ConversationRepo(session).delete_stale(ttl_days)
                        await session.commit()
                        deleted += n
                except Exception as exc:  # noqa: BLE001
                    logger.error("DB prune_stale_conversations failed", error=str(exc))
                    raise

        logger.info("Conversation pruning completed", ttl_days=ttl_days, deleted=deleted)
        return deleted

    async def _prune_file_store(self, cutoff_local: datetime) -> int:
        """Drop entries older than *cutoff_local* from the JSON store.

        Returns how many were pruned (0 if the save failed and the entries were
        rolled back). The caller holds ``_file_mutation_lock``, so no explicit
        delete or clear can interleave with the rollback.
        """
        stale_keys: list[tuple[str, str]] = []
        snapshot: dict[tuple[str, str], Any] = {}
        with self.lock:
            for user_id, user_convs in self._conversations.items():
                for resp_id, payload in user_convs.items():
                    last_activity_raw = payload.get("last_activity")
                    if last_activity_raw:
                        try:
                            la = datetime.fromisoformat(str(last_activity_raw))
                            # Aware timestamps: convert to local naive per-timestamp.
                            # Naive timestamps: already local (written by agent.py).
                            if la.tzinfo is not None:
                                la = la.astimezone().replace(tzinfo=None)
                            if la < cutoff_local:
                                stale_keys.append((user_id, resp_id))
                        except (TypeError, ValueError):
                            pass

            if stale_keys:
                # Snapshot payloads before deletion so we can restore on save failure.
                snapshot = {(uid, rid): self._conversations[uid][rid] for uid, rid in stale_keys}
                for user_id, resp_id in stale_keys:
                    del self._conversations[user_id][resp_id]

        if not stale_keys:
            return 0

        try:
            await self._save_conversations()
        except PostReplaceSyncError as exc:
            # The pruned file is already in place; only its durability is in
            # doubt, so memory must keep matching it. Do not roll back.
            logger.error("Pruned conversation store replaced but not fsynced", error=str(exc))
        except Exception as exc:  # noqa: BLE001
            logger.error("Failed to persist conversation pruning to disk", error=str(exc))
            # Restore deleted entries so in-memory state stays consistent
            # with what is on disk — they were not actually pruned.
            # setdefault: an entry stored again while the save was in flight
            # is newer than our snapshot and must win. Explicit deletes cannot
            # interleave here: they wait on _file_mutation_lock.
            with self.lock:
                for (user_id, resp_id), payload in snapshot.items():
                    self._conversations.setdefault(user_id, {}).setdefault(resp_id, payload)
            return 0
        return len(stale_keys)

    async def clear_user_conversations(self, user_id: str) -> None:
        """Delete every conversation for *user_id* from every active store.

        A file-save failure does not skip the DB delete; the error is re-raised
        afterwards, like ``delete_conversation_thread``. The clear is recorded
        as pending, so a retry saves the file again.
        """
        save_error: Exception | None = None
        if file_writes_enabled():
            # Waits for an in-flight prune, so its rollback cannot undo this.
            async with self._file_mutation_lock:
                with self.lock:
                    removed = self._conversations.pop(user_id, None) is not None
                    if removed:
                        self._pending_file_clears.add(user_id)
                    needs_save = removed or user_id in self._pending_file_clears
                if needs_save:
                    try:
                        await self._save_after_removal()
                    except Exception as exc:  # noqa: BLE001
                        save_error = exc
                        logger.error(
                            "Failed to persist user conversation clear to disk",
                            user_id=user_id,
                            error=str(exc),
                        )

        if db_writes_enabled():
            await self._db_delete_all(user_id)

        if save_error is not None:
            raise save_error

    async def get_storage_stats(self) -> dict[str, Any]:
        # Snapshot — uses whichever storage the mode prioritizes.
        mode = get_storage_mode()
        if mode == "files":
            return {
                "total_users": len(self._conversations),
                "total_conversations": self._count_total(self._conversations),
                "storage_file": self.storage_file,
                "file_exists": os.path.exists(self.storage_file),
            }
        # db / hybrid summary from DB
        try:
            from sqlalchemy import func, select
            from sqlmodel import col

            from db.models import Conversation

            sess_factory = self._resolve_session_factory()
            if sess_factory is None:
                return {"total_users": 0, "total_conversations": 0}
            async with sess_factory() as session:
                total = (
                    await session.execute(select(func.count(col(Conversation.response_id))))
                ).scalar_one()
                users = (
                    await session.execute(select(func.count(func.distinct(Conversation.user_id))))
                ).scalar_one()
            return {
                "total_users": int(users or 0),
                "total_conversations": int(total or 0),
                "storage_file": self.storage_file,
                "file_exists": os.path.exists(self.storage_file),
            }
        except Exception as exc:  # noqa: BLE001
            logger.warning("conversation stats DB read failed", error=str(exc))
            return {"total_users": 0, "total_conversations": 0}

    # ------------------------------------------------------------------
    # Internals
    # ------------------------------------------------------------------

    def _resolve_session_factory(self):
        if self._session_factory is not None:
            return self._session_factory
        try:
            from db.engine import SessionLocal

            return SessionLocal
        except Exception:  # noqa: BLE001
            return None

    @staticmethod
    def _payload_from_row(row) -> dict[str, Any]:
        return {
            "response_id": row.response_id,
            "title": row.title,
            "endpoint": row.endpoint,
            "previous_response_id": row.previous_response_id,
            "filter_id": row.filter_id,
            "total_messages": row.total_messages or 0,
            "created_at": row.created_at.isoformat() if row.created_at else None,
            "last_activity": row.last_activity.isoformat() if row.last_activity else None,
        }

    async def _db_upsert(self, user_id: str, response_id: str, payload: dict[str, Any]) -> None:
        from db.repositories import ConversationRepo

        sess_factory = self._resolve_session_factory()
        if sess_factory is None:
            return
        try:
            async with sess_factory() as session:
                repo = ConversationRepo(session)
                await repo.upsert(
                    response_id=response_id,
                    user_id=user_id,
                    title=payload.get("title"),
                    endpoint=payload.get("endpoint"),
                    previous_response_id=payload.get("previous_response_id"),
                    filter_id=payload.get("filter_id"),
                    total_messages=int(payload.get("total_messages") or 0),
                )
                await session.commit()
        except Exception as exc:  # noqa: BLE001
            logger.error("DB store_conversation failed", error=str(exc))

    async def _db_get_for_user(self, user_id: str) -> dict[str, dict[str, Any]]:
        from db.repositories import ConversationRepo

        sess_factory = self._resolve_session_factory()
        if sess_factory is None:
            return {}
        try:
            async with sess_factory() as session:
                rows = await ConversationRepo(session).list_for_user(user_id)
            return {r.response_id: self._payload_from_row(r) for r in rows}
        except Exception as exc:  # noqa: BLE001
            logger.debug("DB get_for_user failed", error=str(exc))
            return {}

    async def _db_get_one(self, response_id: str, user_id: str) -> dict[str, Any] | None:
        from db.repositories import ConversationRepo

        sess_factory = self._resolve_session_factory()
        if sess_factory is None:
            return None
        try:
            async with sess_factory() as session:
                row = await ConversationRepo(session).get(response_id)
            if row is None or row.user_id != user_id:
                return None
            return self._payload_from_row(row)
        except Exception as exc:  # noqa: BLE001
            logger.debug("DB get_one failed", error=str(exc))
            return None

    async def _db_delete(self, response_id: str, user_id: str) -> bool:
        from db.repositories import ConversationRepo

        sess_factory = self._resolve_session_factory()
        if sess_factory is None:
            return False
        try:
            async with sess_factory() as session:
                ok = await ConversationRepo(session).delete(response_id, user_id)
                await session.commit()
                return ok
        except Exception as exc:  # noqa: BLE001
            logger.error("DB delete failed", error=str(exc))
            return False

    async def _db_delete_all(self, user_id: str) -> int:
        from db.repositories import ConversationRepo

        sess_factory = self._resolve_session_factory()
        if sess_factory is None:
            return 0
        try:
            async with sess_factory() as session:
                n = await ConversationRepo(session).delete_all_for_user(user_id)
                await session.commit()
                return n
        except Exception as exc:  # noqa: BLE001
            logger.error("DB delete_all failed", error=str(exc))
            return 0


# Global instance — session_factory plumbed in main.py at startup.
conversation_persistence = ConversationPersistenceService()