Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

13.5 KB
Raw Blame
"""
Progress Tracker - Tracks knowledge base initialization progress
"""

import asyncio
from collections.abc import Callable
from datetime import datetime
from enum import Enum
import json
import logging
from pathlib import Path

from deeptutor.services.file_io import atomic_write_json

# Use unified logging system

_logger = logging.getLogger(__name__)


def _logger_instance():
    return _logger


def render_message_template(template: str, params: dict[str, object]) -> str:
    """Fill an i18next-style ``{{name}}`` template with *params*.

    Progress lines are shown verbatim in the web log box, so they have to be
    translatable — but the backend has no viewer language (indexing runs as a
    detached task, and the language of record lives in the browser). So the
    wire carries the English template plus its values and the frontend renders
    it with ``t()``; this produces the English fallback for every other
    consumer, from the same single string.
    """
    text = template
    for name, value in params.items():
        text = text.replace("{{" + name + "}}", str(value))
    return text


class ProgressStage(Enum):
    """Initialization stage"""

    INITIALIZING = "initializing"  # Initializing
    PROCESSING_DOCUMENTS = "processing_documents"  # Processing documents
    PROCESSING_FILE = "processing_file"  # Processing single file
    COMPLETED = "completed"  # Completed
    ERROR = "error"  # Error


def visible_progress(progress: dict, kb_dir: Path) -> dict:
    """A prepared native terminal record is visible only after publication."""
    version = progress.get("publication_version")
    if progress.get("stage") != "completed" or not version:
        return progress
    from deeptutor.services.rag.pipelines.lightrag.storage import meta_is_native_published

    if isinstance(version, str) and Path(version).name == version and version not in {".", ".."}:
        if meta_is_native_published(kb_dir / version):
            return progress
    return {
        **progress,
        "stage": "processing_documents",
        "message": "Finalizing index publication",
        "message_key": "Finalizing index publication",
        "message_params": {},
        "progress_percent": 99,
        "index_changed": False,
    }


class ProgressTracker:
    """Progress tracker"""

    def __init__(self, kb_name: str, base_dir: Path):
        self.kb_name = kb_name
        self.base_dir = base_dir
        self.kb_dir = base_dir / kb_name
        self.progress_file = self.kb_dir / ".progress.json"
        self._callbacks: list = []  # Support multiple callbacks
        self.task_id: str | None = None  # Task ID (for log identification)

    def set_callback(self, callback: Callable[[dict], None]):
        """Set progress callback function (can be called multiple times to add multiple callbacks)"""
        if callback not in self._callbacks:
            self._callbacks.append(callback)

    def remove_callback(self, callback: Callable[[dict], None]):
        """Remove progress callback function"""
        if callback in self._callbacks:
            self._callbacks.remove(callback)

    def _notify(self, progress: dict):
        """Notify progress update (call all callbacks)"""
        from deeptutor.runtime.mode import is_server

        if is_server():
            try:
                from deeptutor.knowledge.progress_events import broadcast_progress

                try:
                    loop = asyncio.get_running_loop()
                    loop.create_task(broadcast_progress(self.kb_name, progress))
                except RuntimeError:
                    pass
            except (ImportError, Exception):
                pass

        for callback in self._callbacks:
            try:
                callback(progress)
            except Exception as e:
                _logger_instance().debug("Progress callback error: %s", e)

    def _save_progress(self, progress: dict):
        """Save progress to kb_config.json and local .progress.json file"""
        # Save to kb_config.json (centralized config)
        stored_progress = progress
        try:
            from deeptutor.knowledge.manager import KnowledgeBaseManager

            manager = KnowledgeBaseManager(base_dir=str(self.base_dir))

            # A prepared completion remains processing until its version is published.
            progress = visible_progress(progress, self.kb_dir)
            stage = progress.get("stage", "")
            if stage == "completed":
                status = "ready"
            elif stage == "error":
                status = "error"
            elif stage in [
                "initializing",
                "processing_documents",
                "processing_file",
            ]:
                status = "processing"
            else:
                status = "initializing"

            # Update kb_config.json with status and progress
            manager.update_kb_status(
                name=self.kb_name,
                status=status,
                progress={
                    "stage": progress.get("stage"),
                    "message": progress.get("message"),
                    "percent": progress.get("progress_percent", 0),
                    "current": progress.get("current", 0),
                    "total": progress.get("total", 0),
                    "file_name": progress.get("file_name"),
                    "error": progress.get("error"),
                    "error_code": progress.get("error_code"),
                    "retryable": progress.get("retryable"),
                    "timestamp": progress.get("timestamp"),
                    "task_id": progress.get("task_id"),
                    "indexed_count": progress.get("indexed_count"),
                    "index_changed": progress.get("index_changed"),
                    "index_action": progress.get("index_action"),
                    # Carried alongside the rendered English so a page reload
                    # can still translate the last line it shows.
                    "message_key": progress.get("message_key"),
                    "message_params": progress.get("message_params"),
                },
            )
        except Exception as e:
            _logger_instance().warning("Failed to save progress to kb_config.json: %s", e)

        # Persist the last seen progress snapshot so websocket subscribers and
        # page reloads can recover the live state without relying on in-memory callbacks.
        try:
            atomic_write_json(self.progress_file, stored_progress)
        except Exception as e:
            _logger_instance().warning(
                "Failed to persist progress snapshot for '%s': %s", self.kb_name, e
            )

    def update(
        self,
        stage: ProgressStage,
        message: str = "",
        current: int = 0,
        total: int = 0,
        file_name: str = "",
        error: str | None = None,
        error_code: str | None = None,
        retryable: bool | None = None,
        indexed_count: int | None = None,
        index_changed: bool | None = None,
        index_action: str | None = None,
        message_key: str | None = None,
        message_params: dict[str, object] | None = None,
        publication_version: str | None = None,
    ):
        """Update progress.

        Pass ``message_key`` (an English ``{{name}}`` template) plus
        ``message_params`` instead of a pre-formatted ``message`` so the web log
        box can translate the line; ``message`` is then rendered from the same
        template for every consumer that has no i18n of its own.
        """
        params = message_params or {}
        if message_key and not message:
            message = render_message_template(message_key, params)
        progress: dict[str, object] = {
            "kb_name": self.kb_name,
            "task_id": self.task_id,
            "stage": stage.value,
            "message": message,
            "current": current,
            "total": total,
            "file_name": file_name,
            "progress_percent": int(current / total * 100) if total > 0 else 0,
            "timestamp": datetime.now().isoformat(),
        }
        if indexed_count is not None:
            progress["indexed_count"] = indexed_count
        if index_changed is not None:
            progress["index_changed"] = index_changed
        if index_action:
            progress["index_action"] = index_action
        if message_key:
            progress["message_key"] = message_key
            progress["message_params"] = params

        if error:
            progress["error"] = error
            progress["stage"] = ProgressStage.ERROR.value
        if error_code:
            progress["error_code"] = error_code
        if retryable is not None:
            progress["retryable"] = retryable

        if publication_version is not None:
            progress["publication_version"] = publication_version
        persisted_progress = progress
        progress = visible_progress(progress, self.kb_dir)
        message = str(progress["message"])

        # Output to logger (terminal and log file)
        try:
            logger = _logger_instance()
            prefix = f"[{self.task_id}]" if self.task_id else ""

            if total > 0:
                percent = progress["progress_percent"]
                progress_msg = f"{prefix} {message} ({current}/{total}, {percent}%)"
                if file_name:
                    progress_msg += f" - File: {file_name}"
            else:
                progress_msg = f"{prefix} {message}"
                if file_name:
                    progress_msg += f" - File: {file_name}"

            if error:
                logger.error(f"{progress_msg} - Error: {error}")
            else:
                logger.info(progress_msg)
        except Exception:
            # If unified logging fails unexpectedly, use stdlib logger as fallback.
            fallback_logger = logging.getLogger("deeptutor.ProgressTracker")
            prefix = f"[{self.task_id}]" if self.task_id else ""
            fallback_logger.warning(
                "%s [ProgressTracker] %s (%s/%s)",
                prefix,
                message,
                current,
                total if total > 0 else "?",
            )
            if error:
                fallback_logger.error("%s [ProgressTracker] Error: %s", prefix, error)

        self._save_progress(persisted_progress)

        if self.task_id:
            try:
                from deeptutor.knowledge.progress_events import emit_task_progress

                emit_task_progress(self.task_id, progress)
            except Exception as e:
                _logger_instance().debug("Failed to emit task progress event: %s", e)

        self._notify(progress)

    def verify_terminal(
        self,
        *,
        current: int,
        total: int,
        indexed_count: int,
        index_action: str,
        publication_version: str | None = None,
    ) -> None:
        """Fail publication if either required terminal persistence write was lost."""
        expected = {
            "stage": "completed",
            "task_id": self.task_id,
            "current": current,
            "total": total,
            "indexed_count": indexed_count,
            "index_changed": True,
            "index_action": index_action,
            "progress_percent": 100,
        }
        if publication_version is not None:
            expected["publication_version"] = publication_version
        try:
            with self.progress_file.open(encoding="utf-8") as handle:
                persisted = json.load(handle)
            if not isinstance(persisted, dict) or any(
                persisted.get(key) != value for key, value in expected.items()
            ):
                raise ValueError("Terminal progress does not match this task.")
            from deeptutor.knowledge.manager import KnowledgeBaseManager

            manager = KnowledgeBaseManager(base_dir=str(self.base_dir))
            entry = manager.get_kb_entry(self.kb_name) or {}
            expected_status = "processing" if publication_version else "ready"
            if entry.get("status") != expected_status:
                raise ValueError("Knowledge-base status was not persisted.")
            if publication_version and (entry.get("progress") or {}).get("task_id") != self.task_id:
                raise ValueError("Knowledge-base progress belongs to another task.")
        except Exception as exc:
            raise RuntimeError("Index terminal state was not persisted.") from exc

    def get_progress(self) -> dict | None:
        """Get current progress"""
        if self.progress_file.exists():
            try:
                with open(self.progress_file, encoding="utf-8") as f:
                    return visible_progress(json.load(f), self.kb_dir)
            except Exception as e:
                _logger_instance().debug(f"Failed to read progress file for '{self.kb_name}': {e}")

        try:
            from deeptutor.knowledge.manager import KnowledgeBaseManager

            manager = KnowledgeBaseManager(base_dir=str(self.base_dir))
            status = manager.get_kb_status(self.kb_name)
            if status and status.get("progress"):
                return status.get("progress")
        except Exception as e:
            _logger_instance().debug(
                "Failed to recover progress snapshot from kb_config for '%s': %s",
                self.kb_name,
                e,
            )

        return None

    def clear(self):
        """Clear progress file"""
        if self.progress_file.exists():
            try:
                self.progress_file.unlink()
            except Exception as e:
                _logger_instance().debug(f"Failed to clear progress file for '{self.kb_name}': {e}")