Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

25.9 KB
Raw Blame
"""Parse-server health state and periodic health check task."""

from __future__ import annotations

import asyncio
import errno
import logging
import os
import socket
import subprocess
import sys
import time
from contextlib import contextmanager
from typing import IO, TYPE_CHECKING, Final, Iterator, cast

import httpx

from ...config import LogConfig, ManagedParseServerConfig, get_config
from ...parser.api_client import should_trust_env_for_url
from ...types import MANAGED_SERVER_TIERS, TIERS, ManagedServerTier, Tier
from ...parser.process_control import ManagedProcessControl
from ...utils.stdio import utf8_subprocess_env
from ..config_defaults import CONFIG_DEFAULTS
from ..parse_server_state import ParseServerHealth, ProbeResult, ProbeState
from ..remote_api import resolve_remote_api_key

logger = logging.getLogger("mineru.health_check")

if TYPE_CHECKING:
    from ..services.config_svc import ConfigService

MAX_RESTART_ATTEMPTS = 3
# Local-probe failure backoff: each consecutive failure waits 5s more
# (5/10/15/20/25/30s...), capped at the normal interval, so a server that
# becomes ready during startup is detected quickly.
_FAILURE_BACKOFF_STEP_SEC: Final[int] = 5

_NON_RETRYABLE_MODEL_PRELOAD_ERRORS = frozenset(
    {
        "model_preload_dependency_missing",
        "model_preload_files_missing",
        "model_preload_device_unavailable",
    }
)


async def get_managed_parse_server_tier(config_svc: "ConfigService") -> ManagedServerTier:
    """读取 managed parse-server tier;忽略非法存量值并回退到默认值。"""
    key = "parse_server.local.managed_tier"
    raw_tier = await config_svc.get(key) or CONFIG_DEFAULTS[key]
    if raw_tier in MANAGED_SERVER_TIERS:
        return cast(ManagedServerTier, raw_tier)
    logger.warning(
        "Ignoring invalid managed parse-server tier override %r and using default %r",
        raw_tier,
        CONFIG_DEFAULTS[key],
    )
    return cast(ManagedServerTier, CONFIG_DEFAULTS[key])


def api_server_args_for_tier(tier: ManagedServerTier, *, host: str, port: int) -> list[str]:
    """按统一上限启动托管服务,原生文档解析不受公开档位限制。"""
    return [
        "--tier",
        tier,
        "--host",
        host,
        "--port",
        str(port),
        "--preload-models",
        "--log-level",
        "warning",
    ]


def managed_parse_server_url(host: str, port: int) -> str:
    return f"http://{host}:{port}"


def select_available_managed_port(host: str, port: int, *, strict_port: bool, port_probe_count: int) -> int:
    """按配置探测端口,兼容 Windows 独占绑定及保留端口产生的访问拒绝。"""
    last_exc: OSError | None = None
    candidate_ports = (port,) if strict_port else range(port, port + port_probe_count)
    for candidate_port in candidate_ports:
        probe_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
        try:
            # 与子进程 asyncio.create_server 的默认选项一致,避免 Windows 复用已占用端口后误判可用。
            if os.name == "posix" and sys.platform != "cygwin":
                probe_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
            probe_sock.bind((host, candidate_port))
            return int(probe_sock.getsockname()[1])
        except OSError as exc:
            if strict_port or (exc.errno != errno.EADDRINUSE and getattr(exc, "winerror", None) not in {10013, 10048}):
                raise
            last_exc = exc
        finally:
            probe_sock.close()

    raise RuntimeError(f"No available managed parse-server port in range {port}-{port + port_probe_count - 1}.") from last_exc


def get_parse_server_stdout_log_path() -> str:
    return os.path.expanduser(get_config().doclib.log.resolved_parse_server_stdout_path)


def get_parse_server_stderr_log_path() -> str:
    return os.path.expanduser(get_config().doclib.log.resolved_parse_server_stderr_path)


def _parse_server_stdout_log_path(log_cfg: LogConfig | None) -> str:
    if log_cfg is None:
        return get_parse_server_stdout_log_path()
    return os.path.expanduser(log_cfg.resolved_parse_server_stdout_path)


def _parse_server_stderr_log_path(log_cfg: LogConfig | None) -> str:
    if log_cfg is None:
        return get_parse_server_stderr_log_path()
    return os.path.expanduser(log_cfg.resolved_parse_server_stderr_path)


def _ensure_log_dir(path: str) -> None:
    log_dir = os.path.dirname(path)
    if log_dir:
        os.makedirs(log_dir, exist_ok=True)


@contextmanager
def open_managed_parse_server_logs(*, marker: str, log_cfg: LogConfig | None = None) -> Iterator[tuple[IO[str], IO[str]]]:
    stdout_log_path = _parse_server_stdout_log_path(log_cfg)
    stderr_log_path = _parse_server_stderr_log_path(log_cfg)
    _ensure_log_dir(stdout_log_path)
    _ensure_log_dir(stderr_log_path)

    with (
        open(stdout_log_path, "a", encoding="utf-8") as stdout_log_file,
        open(stderr_log_path, "a", encoding="utf-8") as stderr_log_file,
    ):
        stdout_log_file.write(f"\n--- managed parse-server stdout: {marker} ---\n")
        stderr_log_file.write(f"\n--- managed parse-server stderr: {marker} ---\n")
        stdout_log_file.flush()
        stderr_log_file.flush()
        yield stdout_log_file, stderr_log_file


def start_managed_parse_server(
    *,
    tier: ManagedServerTier,
    managed_cfg: ManagedParseServerConfig,
    log_cfg: LogConfig | None,
    marker: str,
) -> tuple[subprocess.Popen, str, ManagedProcessControl]:
    """创建托管子进程,监听线程或子进程启动失败时回收已创建的控制通道。"""
    port = select_available_managed_port(
        managed_cfg.host,
        managed_cfg.port,
        strict_port=managed_cfg.strict_port,
        port_probe_count=managed_cfg.port_probe_count,
    )
    managed_url = managed_parse_server_url(managed_cfg.host, port)
    cmd = [
        sys.executable,
        "-m",
        "mineru.parser.api_server",
        *api_server_args_for_tier(tier, host=managed_cfg.host, port=port),
    ]
    env = utf8_subprocess_env()
    control = ManagedProcessControl.create()
    env.update(control.child_env())
    logger.info("Starting managed parse-server (%s): %s", marker, " ".join(cmd))
    try:
        control.start_accepting()
        with open_managed_parse_server_logs(marker=marker, log_cfg=log_cfg) as (
            stdout_log_file,
            stderr_log_file,
        ):
            proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=stdout_log_file, stderr=stderr_log_file, env=env)
    except Exception:
        control.close()
        raise
    return proc, managed_url, control


def stop_managed_parse_server(
    proc: subprocess.Popen | None,
    *,
    control: ManagedProcessControl | None = None,
    timeout_sec: int,
    reason: str,
    startup_in_progress: bool = False,
) -> None:
    """在统一预算内关闭托管进程,最终仍未退出时抛错以保留调用方的进程归属。"""
    if proc is None:
        if control is not None:
            control.close()
        return
    if proc.poll() is not None:
        if control is not None:
            control.close()
        return

    pid = proc.pid
    total_timeout = max(float(timeout_sec), 0.0)
    deadline = time.monotonic() + total_timeout
    graceful_cap = min(2.0, total_timeout * 0.2) if startup_in_progress else total_timeout * 0.5
    shutdown_sent = control.request_shutdown(graceful_cap) if control is not None else False
    if shutdown_sent:
        try:
            proc.wait(timeout=min(graceful_cap, max(deadline - time.monotonic(), 0.0)))
            logger.info("Managed parse-server stopped after control shutdown (PID %d, reason=%s)", pid, reason)
            if control is not None:
                control.close()
            return
        except subprocess.TimeoutExpired:
            logger.warning(
                "Managed parse-server did not stop after control shutdown (PID %d, reason=%s), terminating",
                pid,
                reason,
            )
    else:
        logger.warning("Managed parse-server control channel unavailable (PID %d, reason=%s), terminating", pid, reason)

    try:
        proc.terminate()
        terminate_cap = total_timeout * 0.3
        proc.wait(timeout=min(terminate_cap, max(deadline - time.monotonic(), 0.0)))
        logger.info("Managed parse-server terminated (PID %d, reason=%s)", pid, reason)
        if control is not None:
            control.close()
        return
    except subprocess.TimeoutExpired:
        logger.warning("Managed parse-server did not terminate within timeout (PID %d, reason=%s), killing", pid, reason)
    except Exception as exc:
        logger.warning("Failed to terminate managed parse-server (PID %d, reason=%s): %s", pid, reason, exc)

    try:
        proc.kill()
        proc.wait(timeout=max(deadline - time.monotonic(), 0.0))
        logger.info("Managed parse-server killed (PID %d, reason=%s)", pid, reason)
    except Exception as exc:
        logger.error("Failed to kill managed parse-server (PID %d, reason=%s): %s", pid, reason, exc)
        if proc.poll() is None:
            raise RuntimeError(f"Managed parse-server did not stop (PID {pid}, reason={reason})") from exc
    finally:
        if control is not None:
            control.close()


def _managed_parse_server_needs_restart(
    health: ParseServerHealth,
    *,
    now: float,
    startup_timeout_sec: int,
) -> bool:
    if health.local.probe.error_code in _NON_RETRYABLE_MODEL_PRELOAD_ERRORS:
        return False
    proc = health.managed_proc
    if health.local_starting and proc is not None and proc.poll() is None:
        return now - health.local_started_at >= startup_timeout_sec
    return True


class ParseServerHealthCheck:
    """Periodically probes parse-server health via GET /v1/tiers."""

    def __init__(
        self,
        config_svc: "ConfigService",
        *,
        interval_sec: int,
        probe_timeout_sec: int,
        startup_grace_sec: int,
        stop_timeout_sec: int,
        health: ParseServerHealth,
        startup_timeout_sec: int,
        managed_parse_server: ManagedParseServerConfig,
        log_cfg: LogConfig,
    ) -> None:
        """绑定应用健康状态与启动配置,不回落到进程默认配置。"""
        self.health = health
        self.config_svc = config_svc
        self.interval_sec = interval_sec
        self.probe_timeout_sec = probe_timeout_sec
        self.startup_grace_sec = startup_grace_sec
        self.startup_timeout_sec = startup_timeout_sec
        self.stop_timeout_sec = stop_timeout_sec
        self.managed_parse_server = managed_parse_server
        self.log_cfg = log_cfg
        self.running = False
        self._wakeup_event = asyncio.Event()

    def wakeup(self) -> None:
        """配置变更后立即唤醒健康检查,不等待正常探测间隔。"""
        self._wakeup_event.set()

    async def run(self) -> None:
        self.running = True
        health = self.health
        consecutive_local_failures = 0

        while self.running:
            self._wakeup_event.clear()
            config_generation = health.local_generation
            # refresh config on each cycle (hot-reload)
            mode = (await self.config_svc.get("parse_server.local.mode")) or "disabled"
            desired_managed_tier = await get_managed_parse_server_tier(self.config_svc)
            self_hosted_url = await self.config_svc.get("parse_server.local.self_hosted_url")
            # 读取配置期间若发生切换,重新读取,避免用旧快照覆盖新的目标档位。
            if config_generation != health.local_generation:
                continue
            if mode == "managed" and (health.local_mode != mode or health.managed_tier != desired_managed_tier):
                health.restart_count = 0
                health.managed_restart_exhausted = False
            health.local_mode = mode
            health.managed_tier = desired_managed_tier
            health.self_hosted_url = self_hosted_url if self_hosted_url else None

            # 档位切换先于探测和远程健康检查,旧服务不能在重启前重新变为可用。
            tier_changed = False
            if health.local_mode == "managed" and not health.managed_restart_exhausted:
                tier_changed = await self._try_restart_managed_for_tier_change(health, desired_managed_tier)
            elif health.local_mode != "managed" and (health.managed_proc is not None or health.managed_control is not None):
                try:
                    self._stop_managed_process(health, reason="local mode changed")
                except RuntimeError:
                    logger.exception("Failed to stop managed parse-server after local mode changed")

            # probe local
            probed_local = False
            probe_generation = health.local_generation
            if health.local_mode == "managed" and health.managed_restart_exhausted:
                self._exhaust_managed_restarts(health)
            elif health.local_mode != "disabled":
                url = self._local_url(health)
                if url:
                    probed_local = True
                    api_key = None
                    if health.local_mode == "self_hosted":
                        api_key = (await self.config_svc.get("parse_server.local.self_hosted_api_key")) or None
                    probe = await self._probe(url, api_key=api_key)
                    if probe_generation != health.local_generation:
                        continue
                    if health.local_mode == "managed" and probe.healthy:
                        if health.running_managed_tier != health.managed_tier or health.managed_tier not in probe.tiers:
                            probe = ProbeResult(
                                error_code="parse_server_unavailable",
                                error_msg=f"Managed parse-server is not ready for tier '{health.managed_tier}'.",
                            )
                    now_ms = int(time.time() * 1000)
                    health.local.url = url
                    health.local.probe = probe
                    health.local.last_probe_at = now_ms
                    if probe.healthy:
                        health.local.last_success_at = now_ms
                        health.local_starting = False
                        health.managed_tier_transition = False
                        # MAX_RESTART_ATTEMPTS bounds consecutive failed recoveries; a healthy probe restores the budget.
                        health.restart_count = 0
                    else:
                        health.local.last_failure_at = now_ms
                        if probe.error_code != "parse_server_unavailable":
                            health.local_starting = False
                            health.managed_tier_transition = False
                else:
                    health.local = ProbeState()
            else:
                health.local = ProbeState()

            # probe remote
            remote_url = cast(str, await self.config_svc.get("parse_server.remote.url"))
            remote_api_key = (await resolve_remote_api_key(self.config_svc)).value
            probe = await self._probe(remote_url, api_key=remote_api_key)
            now_ms = int(time.time() * 1000)
            health.remote.url = remote_url
            health.remote.probe = probe
            health.remote.last_probe_at = now_ms
            if probe.healthy:
                health.remote.last_success_at = now_ms
            else:
                health.remote.last_failure_at = now_ms

            if probe_generation != health.local_generation:
                continue
            if health.local_mode == "managed" and not health.managed_restart_exhausted:
                proc = health.managed_proc
                now = asyncio.get_event_loop().time()
                startup_elapsed_sec = now - health.local_started_at
                if (
                    not tier_changed
                    and health.local_starting
                    and proc is not None
                    and proc.poll() is None
                    and startup_elapsed_sec >= self.startup_grace_sec
                    and startup_elapsed_sec < self.startup_timeout_sec
                ):
                    logger.info(
                        "Managed parse-server is still preparing startup tier %s (PID %d)",
                        health.running_managed_tier,
                        proc.pid,
                    )
                if (
                    not tier_changed
                    and not health.local.probe.healthy
                    and _managed_parse_server_needs_restart(
                        health,
                        now=now,
                        startup_timeout_sec=self.startup_timeout_sec,
                    )
                ):
                    if health.local_starting and proc is not None and proc.poll() is None:
                        logger.warning(
                            "Managed parse-server startup exceeded %ds for tier %s (PID %d), attempting restart",
                            self.startup_timeout_sec,
                            health.running_managed_tier,
                            proc.pid,
                        )
                    else:
                        logger.warning("Managed parse-server is unhealthy, attempting restart")
                    await self._try_restart_managed(health)

            # A failed local probe shortens the wait (5/10/20s, capped at the
            # normal interval) so a server that becomes ready during startup is
            # detected quickly; any success or non-probed cycle restores it.
            if probed_local and not health.local.probe.healthy:
                consecutive_local_failures += 1
            else:
                consecutive_local_failures = 0
            if self.running:
                try:
                    await asyncio.wait_for(
                        self._wakeup_event.wait(), timeout=self._next_interval_sec(consecutive_local_failures)
                    )
                except asyncio.TimeoutError:
                    pass

    def _next_interval_sec(self, consecutive_local_failures: int) -> int:
        """下一次探测的等待秒数;每多失败一次多等 5 秒,封顶正常间隔。"""
        if consecutive_local_failures <= 0:
            return self.interval_sec
        return min(consecutive_local_failures * _FAILURE_BACKOFF_STEP_SEC, self.interval_sec)

    async def _try_restart_managed_for_tier_change(
        self,
        health: ParseServerHealth,
        desired_managed_tier: ManagedServerTier,
    ) -> bool:
        """目标档位变化时串行替换托管进程,熔断恢复由配置入口显式触发。"""
        if health.managed_restart_exhausted:
            return False
        running_managed_tier = health.running_managed_tier
        if running_managed_tier is None or running_managed_tier == desired_managed_tier:
            return False

        logger.info(
            "Managed parse-server tier changed from %s to %s, restarting",
            running_managed_tier,
            desired_managed_tier,
        )
        await self._try_restart_managed(
            health,
            reason="tier-change",
            marker=f"tier change {running_managed_tier}->{desired_managed_tier}",
            count_restart=False,
        )
        return True

    async def _try_restart_managed(
        self,
        health: ParseServerHealth,
        *,
        reason: str = "restart",
        marker: str | None = None,
        count_restart: bool = True,
    ) -> None:
        """先确认旧进程已退出再启动新进程,连续失败达到预算后进入熔断。"""
        if health.managed_restart_exhausted or (count_restart and health.restart_count >= MAX_RESTART_ATTEMPTS):
            self._exhaust_managed_restarts(health)
            return

        config_generation = health.local_generation
        managed_tier = await get_managed_parse_server_tier(self.config_svc)
        if config_generation != health.local_generation:
            return
        if count_restart:
            health.restart_count += 1
        health.begin_managed_tier_transition(managed_tier)
        try:
            self._stop_managed_process(health, reason=reason)
            proc, managed_url, control = start_managed_parse_server(
                tier=managed_tier,
                managed_cfg=self.managed_parse_server,
                log_cfg=self.log_cfg,
                marker=marker or f"restart attempt {health.restart_count}",
            )
            health.managed_url = managed_url
            logger.info("Managed parse-server restarted (PID %d, tier=%s)", proc.pid, managed_tier)
            health.managed_proc = proc
            health.managed_control = control
            health.running_managed_tier = managed_tier
            health.local_starting = True
            health.local_started_at = asyncio.get_event_loop().time()
        except Exception as exc:
            logger.error(f"Failed to restart managed parse-server: {exc}")
            # 配置切换成功不消耗恢复预算,但切换失败仍须受连续失败上限约束。
            if not count_restart:
                health.restart_count += 1
            if health.restart_count >= MAX_RESTART_ATTEMPTS:
                self._exhaust_managed_restarts(health)

    def _stop_managed_process(self, health: ParseServerHealth, *, reason: str) -> None:
        """成功回收后释放进程归属,失败时保留引用供下一轮或应用退出重试。"""
        stop_managed_parse_server(
            health.managed_proc,
            control=health.managed_control,
            timeout_sec=self.stop_timeout_sec,
            reason=reason,
            startup_in_progress=health.local_starting,
        )
        health.managed_proc = None
        health.managed_control = None
        health.running_managed_tier = None
        health.local_starting = False
        health.local_started_at = 0.0

    def _exhaust_managed_restarts(self, health: ParseServerHealth) -> None:
        """锁定失败恢复并回收残留进程,后续轮次只重试未完成的回收。"""
        if not health.managed_restart_exhausted:
            logger.error("Managed parse-server failed %d restarts, disabling automatic restarts", MAX_RESTART_ATTEMPTS)
            health.managed_restart_exhausted = True
            health.local_generation += 1
            health.local.probe = ProbeResult(
                error_code="parse_server_unavailable",
                error_msg=(
                    f"Managed parse-server failed {MAX_RESTART_ATTEMPTS} restarts. "
                    "Set managed mode or tier again, or restart MinerU server to retry."
                ),
            )
        try:
            if health.managed_proc is not None or health.managed_control is not None:
                self._stop_managed_process(health, reason="restart budget exhausted")
        except RuntimeError:
            logger.exception("Failed to stop managed parse-server after restart budget exhausted")
        health.local_starting = False
        health.managed_tier_transition = False

    async def _probe(self, base_url: str, *, api_key: str | None = None) -> ProbeResult:
        headers = {"Authorization": f"Bearer {api_key}"} if api_key else None
        try:
            async with httpx.AsyncClient(
                timeout=self.probe_timeout_sec,
                trust_env=should_trust_env_for_url(base_url),
            ) as client:
                resp = await client.get(f"{base_url}/v1/tiers", headers=headers)
                if resp.status_code == 200:
                    data = resp.json()
                    tiers: list[Tier] = []
                    for t in data.get("data", []):
                        tier_id = t.get("id")
                        if tier_id in TIERS:
                            tiers.append(cast(Tier, tier_id))
                    return ProbeResult(healthy=True, tiers=tiers)
                error_code, error_msg = _probe_error(resp)
                return ProbeResult(error_code=error_code, error_msg=error_msg)
        except httpx.TimeoutException as exc:
            return ProbeResult(error_code="parse_server_unavailable", error_msg=str(exc) or "Parse-server probe timed out.")
        except httpx.TransportError as exc:
            return ProbeResult(
                error_code="parse_server_unavailable",
                error_msg=str(exc) or "Parse-server probe transport failed.",
            )
        except Exception as exc:
            return ProbeResult(error_code="parse_server_unavailable", error_msg=str(exc) or "Parse-server probe failed.")

    @staticmethod
    def _local_url(health: ParseServerHealth) -> str | None:
        if health.local_mode == "managed":
            return health.managed_url
        if health.local_mode == "self_hosted":
            return health.self_hosted_url
        return None

    async def stop(self) -> None:
        """结束健康检查并确认回收当前应用拥有的托管进程。"""
        self.running = False
        self.wakeup()
        health = self.health
        self._stop_managed_process(health, reason="health check stop")


def _probe_error(resp: httpx.Response) -> tuple[str, str]:
    data: dict[str, object] = {}
    try:
        loaded = resp.json()
        if isinstance(loaded, dict):
            data = loaded
    except Exception:
        data = {}

    error = data.get("error")
    if isinstance(error, dict):
        code = str(error.get("code") or "http_error")
        message = str(error.get("message") or error)
        return code, message

    remote_message = _remote_auth_message(data)
    if resp.status_code == 401 or remote_message is not None:
        return "invalid_api_key", remote_message or "API key invalid or remote authentication failed."

    text = resp.text[:500]
    return "http_error", f"HTTP {resp.status_code}: {text}"


def _remote_auth_message(data: dict[str, object]) -> str | None:
    msg_code = data.get("msgCode")
    msg = data.get("msg")
    if msg_code == "A0202":
        return str(msg or "user authenticate failed")
    if isinstance(msg, str) and "authenticate failed" in msg.lower():
        return msg
    return None