DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
"""ParseService — the public entry to the document-parse bridge.
Resolves the active (or requested) engine, computes the content-addressed cache
key, returns a cached :class:`ParsedDocument` on hit, and on miss runs the
engine behind the model-download readiness gate, then caches and returns the IR.
Consumption is pull-based: callers ask for a parse when they need structured
content; nothing is parsed implicitly.
"""
from __future__ import annotations
import logging
from pathlib import Path
from typing import Any, Callable, Optional
from deeptutor.services.config.runtime_settings import (
_DEFAULT_DOCUMENT_PARSING_ENGINE,
load_document_parsing_settings,
)
from . import cache
from .engines.factory import get_parser
from .types import ParsedDocument, ParserError
logger = logging.getLogger(__name__)
def _matches_supported_format(source_path: str | Path, supported: frozenset[str]) -> bool:
"""Match simple or compound parser suffixes case-insensitively."""
name = Path(source_path).name.lower()
return any(name.endswith(str(extension).lower()) for extension in supported)
def _display_extension(source_path: str | Path, supported: frozenset[str]) -> str:
"""Return the most specific advertised suffix for an error message."""
name = Path(source_path).name.lower()
matches = [extension for extension in supported if name.endswith(extension.lower())]
if matches:
return max(matches, key=len)
return Path(source_path).suffix.lower() or "this"
# Engines tried, in order, when the *implicitly* selected engine (the global
# default) does not advertise support for a file's suffix. The global
# single-select is a default, not a per-file command: a mixed KB (MinerU PDFs
# plus .txt/.md notes) must not hard-fail the whole ingest batch on the files
# the chosen engine was never built for (#1502). ``text_only`` leads because
# it is lossless for text files and needs no model.
_FALLBACK_ENGINE_PREFERENCE = ("text_only", "markitdown", "tika")
def _find_fallback_engine(
source_path: str | Path, primary_engine: str
) -> Optional[tuple[str, Any, Any]]:
"""First installed engine other than ``primary`` that supports the file.
Returns ``(engine_name, parser, config)`` or ``None``. Engines that are
not installed are skipped — the fallback must never turn a routing
decision into an import-time failure.
"""
for name in _FALLBACK_ENGINE_PREFERENCE:
if name == primary_engine:
continue
try:
parser = get_parser(name)
except Exception:
continue
supported = parser.supported_formats()
if not supported or _matches_supported_format(source_path, supported):
return name, parser, parser.resolve_config()
return None
class ParseService:
"""Cache-aware, engine-pluggable document parsing."""
def __init__(self, cache_root: Optional[Path] = None) -> None:
self._cache_root_override = Path(cache_root) if cache_root else None
def _cache_root(self) -> Path:
if self._cache_root_override is not None:
return self._cache_root_override
# Resolve lazily per call so the cache root tracks the active
# user/workspace (multi-user safety), like get_path_service().
from deeptutor.services.path_service import get_path_service
return get_path_service().get_parse_cache_root()
def active_engine(self) -> str:
return str(
load_document_parsing_settings().get("engine") or _DEFAULT_DOCUMENT_PARSING_ENGINE
)
def supports(self, source_path: str | Path, *, engine: Optional[str] = None) -> bool:
"""Return whether the selected engine advertises support for this path.
This is a cheap routing check only: it does not require the file to
exist, initialize models, or evaluate engine readiness.
"""
engine_name = (engine or self.active_engine()).strip().lower()
supported = get_parser(engine_name).supported_formats()
return not supported or _matches_supported_format(source_path, supported)
def parse(self, source_path: str | Path, *, engine=None, on_output=None) -> ParsedDocument:
from deeptutor.services.parsing.progress import _redact, current_run, failure_code
run = current_run()
source = Path(source_path)
if run:
run.phase("parsing", source=source)
run.document(source, "parsing")
def output(line):
if run:
run.phase("parsing", source=source, activity=line)
if on_output:
on_output(line)
try:
result = self._parse(source, engine=engine, on_output=output)
if run:
run.check()
run.document(
source, "parsed", parser=result.engine, parser_signature=result.parser_signature
)
return result
except Exception as exc:
if run:
run.document(source, "failed", reason=failure_code(exc), detail=_redact(str(exc)))
raise
def _parse(
self,
source_path: str | Path,
*,
engine: Optional[str] = None,
on_output: Optional[Callable[[str], None]] = None,
) -> ParsedDocument:
"""Parse ``source_path`` with the active (or requested) engine.
Returns a cached result when one exists for the same bytes + engine
signature. Raises :class:`ParserError` when the engine is not ready
(e.g. local models not downloaded and auto-download disabled) or the
file type is unsupported.
"""
source_path = Path(source_path)
if not source_path.is_file():
raise ParserError(f"File to parse not found: {source_path}")
engine_name = (engine or self.active_engine()).strip().lower()
parser = get_parser(engine_name)
config = parser.resolve_config()
supported = parser.supported_formats()
if supported and not _matches_supported_format(source_path, supported):
if engine is not None:
# The caller pinned this engine by name — keep the loud error.
suffix = _display_extension(source_path, supported)
raise ParserError(
f"The '{engine_name}' parsing engine doesn't support {suffix} "
f"files. Choose a different engine in Settings → Document Parsing."
)
fallback = _find_fallback_engine(source_path, engine_name)
if fallback is None:
suffix = _display_extension(source_path, supported)
raise ParserError(
f"The '{engine_name}' parsing engine doesn't support {suffix} "
f"files. Choose a different engine in Settings → Document Parsing."
)
primary_engine = engine_name
engine_name, parser, config = fallback
supported = parser.supported_formats()
logger.warning(
"The '%s' parsing engine doesn't support %s files; using '%s' for this file.",
primary_engine,
_display_extension(source_path, supported),
engine_name,
)
sig = parser.signature(config).hash()
source_hash = cache.source_hash_from_path(source_path)
cache_root = self._cache_root()
hit = cache.lookup(cache_root, source_hash, sig)
if hit is not None:
try:
markdown, blocks, asset_dir = cache.load_ir(hit)
except (OSError, UnicodeError, ValueError):
markdown, blocks, asset_dir = "", None, None
if markdown.strip() or blocks:
logger.info("Parse cache hit for %s (%s/%s)", source_path.name, engine_name, sig)
from deeptutor.services.parsing.progress import current_run
if run := current_run():
run.document(source_path, "parsed", parse_reused=True)
return ParsedDocument(
markdown=markdown,
blocks=blocks,
asset_dir=asset_dir,
source_hash=source_hash,
parser_signature=sig,
engine=engine_name,
workdir=hit,
)
# A ready stamp alone cannot make missing/empty output usable (#1612).
logger.warning("Invalid parse cache for %s; restarting this document", source_path.name)
(hit / cache.MANIFEST_FILENAME).unlink(missing_ok=True)
report = parser.is_ready(config)
if not report.ready:
raise ParserError(report.message or f"The '{engine_name}' engine is not ready.")
workdir = cache.reserve(cache_root, source_hash, sig)
logger.info("Parsing %s with %s (signature %s)", source_path.name, engine_name, sig)
try:
parser.parse(source_path, workdir, config=config, on_output=on_output)
markdown, blocks, asset_dir = cache.load_ir(workdir)
if not markdown.strip() and not blocks:
raise ParserError(
f"The '{engine_name}' engine produced no content for {source_path.name}."
)
cache.write_manifest(
workdir,
{
"engine": engine_name,
"signature": sig,
"source_hash": source_hash,
"source_name": source_path.name,
},
)
return ParsedDocument(
markdown=markdown,
blocks=blocks,
asset_dir=asset_dir,
source_hash=source_hash,
parser_signature=sig,
engine=engine_name,
workdir=workdir,
)
except Exception:
cache.cleanup_failed(workdir)
raise
_service: Optional[ParseService] = None
def get_parse_service() -> ParseService:
"""Return the process-wide :class:`ParseService` singleton."""
global _service
if _service is None:
_service = ParseService()
return _service
__all__ = ["ParseService", "get_parse_service"]