Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

10.1 KB
Raw Blame
"""ParseService — the public entry to the document-parse bridge.

Resolves the active (or requested) engine, computes the content-addressed cache
key, returns a cached :class:`ParsedDocument` on hit, and on miss runs the
engine behind the model-download readiness gate, then caches and returns the IR.
Consumption is pull-based: callers ask for a parse when they need structured
content; nothing is parsed implicitly.
"""

from __future__ import annotations

import logging
from pathlib import Path
from typing import Any, Callable, Optional

from deeptutor.services.config.runtime_settings import (
    _DEFAULT_DOCUMENT_PARSING_ENGINE,
    load_document_parsing_settings,
)

from . import cache
from .engines.factory import get_parser
from .types import ParsedDocument, ParserError

logger = logging.getLogger(__name__)


def _matches_supported_format(source_path: str | Path, supported: frozenset[str]) -> bool:
    """Match simple or compound parser suffixes case-insensitively."""
    name = Path(source_path).name.lower()
    return any(name.endswith(str(extension).lower()) for extension in supported)


def _display_extension(source_path: str | Path, supported: frozenset[str]) -> str:
    """Return the most specific advertised suffix for an error message."""
    name = Path(source_path).name.lower()
    matches = [extension for extension in supported if name.endswith(extension.lower())]
    if matches:
        return max(matches, key=len)
    return Path(source_path).suffix.lower() or "this"


# Engines tried, in order, when the *implicitly* selected engine (the global
# default) does not advertise support for a file's suffix. The global
# single-select is a default, not a per-file command: a mixed KB (MinerU PDFs
# plus .txt/.md notes) must not hard-fail the whole ingest batch on the files
# the chosen engine was never built for (#1502). ``text_only`` leads because
# it is lossless for text files and needs no model.
_FALLBACK_ENGINE_PREFERENCE = ("text_only", "markitdown", "tika")


def _find_fallback_engine(
    source_path: str | Path, primary_engine: str
) -> Optional[tuple[str, Any, Any]]:
    """First installed engine other than ``primary`` that supports the file.

    Returns ``(engine_name, parser, config)`` or ``None``. Engines that are
    not installed are skipped — the fallback must never turn a routing
    decision into an import-time failure.
    """
    for name in _FALLBACK_ENGINE_PREFERENCE:
        if name == primary_engine:
            continue
        try:
            parser = get_parser(name)
        except Exception:
            continue
        supported = parser.supported_formats()
        if not supported or _matches_supported_format(source_path, supported):
            return name, parser, parser.resolve_config()
    return None


class ParseService:
    """Cache-aware, engine-pluggable document parsing."""

    def __init__(self, cache_root: Optional[Path] = None) -> None:
        self._cache_root_override = Path(cache_root) if cache_root else None

    def _cache_root(self) -> Path:
        if self._cache_root_override is not None:
            return self._cache_root_override
        # Resolve lazily per call so the cache root tracks the active
        # user/workspace (multi-user safety), like get_path_service().
        from deeptutor.services.path_service import get_path_service

        return get_path_service().get_parse_cache_root()

    def active_engine(self) -> str:
        return str(
            load_document_parsing_settings().get("engine") or _DEFAULT_DOCUMENT_PARSING_ENGINE
        )

    def supports(self, source_path: str | Path, *, engine: Optional[str] = None) -> bool:
        """Return whether the selected engine advertises support for this path.

        This is a cheap routing check only: it does not require the file to
        exist, initialize models, or evaluate engine readiness.
        """
        engine_name = (engine or self.active_engine()).strip().lower()
        supported = get_parser(engine_name).supported_formats()
        return not supported or _matches_supported_format(source_path, supported)

    def parse(self, source_path: str | Path, *, engine=None, on_output=None) -> ParsedDocument:
        from deeptutor.services.parsing.progress import _redact, current_run, failure_code

        run = current_run()
        source = Path(source_path)
        if run:
            run.phase("parsing", source=source)
            run.document(source, "parsing")

        def output(line):
            if run:
                run.phase("parsing", source=source, activity=line)
            if on_output:
                on_output(line)

        try:
            result = self._parse(source, engine=engine, on_output=output)
            if run:
                run.check()
                run.document(
                    source, "parsed", parser=result.engine, parser_signature=result.parser_signature
                )
            return result
        except Exception as exc:
            if run:
                run.document(source, "failed", reason=failure_code(exc), detail=_redact(str(exc)))
            raise

    def _parse(
        self,
        source_path: str | Path,
        *,
        engine: Optional[str] = None,
        on_output: Optional[Callable[[str], None]] = None,
    ) -> ParsedDocument:
        """Parse ``source_path`` with the active (or requested) engine.

        Returns a cached result when one exists for the same bytes + engine
        signature. Raises :class:`ParserError` when the engine is not ready
        (e.g. local models not downloaded and auto-download disabled) or the
        file type is unsupported.
        """
        source_path = Path(source_path)
        if not source_path.is_file():
            raise ParserError(f"File to parse not found: {source_path}")

        engine_name = (engine or self.active_engine()).strip().lower()
        parser = get_parser(engine_name)
        config = parser.resolve_config()

        supported = parser.supported_formats()
        if supported and not _matches_supported_format(source_path, supported):
            if engine is not None:
                # The caller pinned this engine by name — keep the loud error.
                suffix = _display_extension(source_path, supported)
                raise ParserError(
                    f"The '{engine_name}' parsing engine doesn't support {suffix} "
                    f"files. Choose a different engine in Settings → Document Parsing."
                )
            fallback = _find_fallback_engine(source_path, engine_name)
            if fallback is None:
                suffix = _display_extension(source_path, supported)
                raise ParserError(
                    f"The '{engine_name}' parsing engine doesn't support {suffix} "
                    f"files. Choose a different engine in Settings → Document Parsing."
                )
            primary_engine = engine_name
            engine_name, parser, config = fallback
            supported = parser.supported_formats()
            logger.warning(
                "The '%s' parsing engine doesn't support %s files; using '%s' for this file.",
                primary_engine,
                _display_extension(source_path, supported),
                engine_name,
            )

        sig = parser.signature(config).hash()
        source_hash = cache.source_hash_from_path(source_path)
        cache_root = self._cache_root()

        hit = cache.lookup(cache_root, source_hash, sig)
        if hit is not None:
            try:
                markdown, blocks, asset_dir = cache.load_ir(hit)
            except (OSError, UnicodeError, ValueError):
                markdown, blocks, asset_dir = "", None, None
            if markdown.strip() or blocks:
                logger.info("Parse cache hit for %s (%s/%s)", source_path.name, engine_name, sig)
                from deeptutor.services.parsing.progress import current_run

                if run := current_run():
                    run.document(source_path, "parsed", parse_reused=True)
                return ParsedDocument(
                    markdown=markdown,
                    blocks=blocks,
                    asset_dir=asset_dir,
                    source_hash=source_hash,
                    parser_signature=sig,
                    engine=engine_name,
                    workdir=hit,
                )
            # A ready stamp alone cannot make missing/empty output usable (#1612).
            logger.warning("Invalid parse cache for %s; restarting this document", source_path.name)
            (hit / cache.MANIFEST_FILENAME).unlink(missing_ok=True)

        report = parser.is_ready(config)
        if not report.ready:
            raise ParserError(report.message or f"The '{engine_name}' engine is not ready.")

        workdir = cache.reserve(cache_root, source_hash, sig)
        logger.info("Parsing %s with %s (signature %s)", source_path.name, engine_name, sig)
        try:
            parser.parse(source_path, workdir, config=config, on_output=on_output)
            markdown, blocks, asset_dir = cache.load_ir(workdir)
            if not markdown.strip() and not blocks:
                raise ParserError(
                    f"The '{engine_name}' engine produced no content for {source_path.name}."
                )
            cache.write_manifest(
                workdir,
                {
                    "engine": engine_name,
                    "signature": sig,
                    "source_hash": source_hash,
                    "source_name": source_path.name,
                },
            )
            return ParsedDocument(
                markdown=markdown,
                blocks=blocks,
                asset_dir=asset_dir,
                source_hash=source_hash,
                parser_signature=sig,
                engine=engine_name,
                workdir=workdir,
            )
        except Exception:
            cache.cleanup_failed(workdir)
            raise


_service: Optional[ParseService] = None


def get_parse_service() -> ParseService:
    """Return the process-wide :class:`ParseService` singleton."""
    global _service
    if _service is None:
        _service = ParseService()
    return _service


__all__ = ["ParseService", "get_parse_service"]