Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
"""Parse-server health state and periodic health check task."""
from __future__ import annotations
import asyncio
import errno
import logging
import os
import socket
import subprocess
import sys
import time
from contextlib import contextmanager
from typing import IO, TYPE_CHECKING, Final, Iterator, cast
import httpx
from ...config import LogConfig, ManagedParseServerConfig, get_config
from ...parser.api_client import should_trust_env_for_url
from ...types import MANAGED_SERVER_TIERS, TIERS, ManagedServerTier, Tier
from ...parser.process_control import ManagedProcessControl
from ...utils.stdio import utf8_subprocess_env
from ..config_defaults import CONFIG_DEFAULTS
from ..parse_server_state import ParseServerHealth, ProbeResult, ProbeState
from ..remote_api import resolve_remote_api_key
logger = logging.getLogger("mineru.health_check")
if TYPE_CHECKING:
from ..services.config_svc import ConfigService
MAX_RESTART_ATTEMPTS = 3
# Local-probe failure backoff: each consecutive failure waits 5s more
# (5/10/15/20/25/30s...), capped at the normal interval, so a server that
# becomes ready during startup is detected quickly.
_FAILURE_BACKOFF_STEP_SEC: Final[int] = 5
_NON_RETRYABLE_MODEL_PRELOAD_ERRORS = frozenset(
{
"model_preload_dependency_missing",
"model_preload_files_missing",
"model_preload_device_unavailable",
}
)
async def get_managed_parse_server_tier(config_svc: "ConfigService") -> ManagedServerTier:
"""读取 managed parse-server tier;忽略非法存量值并回退到默认值。"""
key = "parse_server.local.managed_tier"
raw_tier = await config_svc.get(key) or CONFIG_DEFAULTS[key]
if raw_tier in MANAGED_SERVER_TIERS:
return cast(ManagedServerTier, raw_tier)
logger.warning(
"Ignoring invalid managed parse-server tier override %r and using default %r",
raw_tier,
CONFIG_DEFAULTS[key],
)
return cast(ManagedServerTier, CONFIG_DEFAULTS[key])
def api_server_args_for_tier(tier: ManagedServerTier, *, host: str, port: int) -> list[str]:
"""按统一上限启动托管服务,原生文档解析不受公开档位限制。"""
return [
"--tier",
tier,
"--host",
host,
"--port",
str(port),
"--preload-models",
"--log-level",
"warning",
]
def managed_parse_server_url(host: str, port: int) -> str:
return f"http://{host}:{port}"
def select_available_managed_port(host: str, port: int, *, strict_port: bool, port_probe_count: int) -> int:
"""按配置探测端口,兼容 Windows 独占绑定及保留端口产生的访问拒绝。"""
last_exc: OSError | None = None
candidate_ports = (port,) if strict_port else range(port, port + port_probe_count)
for candidate_port in candidate_ports:
probe_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
# 与子进程 asyncio.create_server 的默认选项一致,避免 Windows 复用已占用端口后误判可用。
if os.name == "posix" and sys.platform != "cygwin":
probe_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
probe_sock.bind((host, candidate_port))
return int(probe_sock.getsockname()[1])
except OSError as exc:
if strict_port or (exc.errno != errno.EADDRINUSE and getattr(exc, "winerror", None) not in {10013, 10048}):
raise
last_exc = exc
finally:
probe_sock.close()
raise RuntimeError(f"No available managed parse-server port in range {port}-{port + port_probe_count - 1}.") from last_exc
def get_parse_server_stdout_log_path() -> str:
return os.path.expanduser(get_config().doclib.log.resolved_parse_server_stdout_path)
def get_parse_server_stderr_log_path() -> str:
return os.path.expanduser(get_config().doclib.log.resolved_parse_server_stderr_path)
def _parse_server_stdout_log_path(log_cfg: LogConfig | None) -> str:
if log_cfg is None:
return get_parse_server_stdout_log_path()
return os.path.expanduser(log_cfg.resolved_parse_server_stdout_path)
def _parse_server_stderr_log_path(log_cfg: LogConfig | None) -> str:
if log_cfg is None:
return get_parse_server_stderr_log_path()
return os.path.expanduser(log_cfg.resolved_parse_server_stderr_path)
def _ensure_log_dir(path: str) -> None:
log_dir = os.path.dirname(path)
if log_dir:
os.makedirs(log_dir, exist_ok=True)
@contextmanager
def open_managed_parse_server_logs(*, marker: str, log_cfg: LogConfig | None = None) -> Iterator[tuple[IO[str], IO[str]]]:
stdout_log_path = _parse_server_stdout_log_path(log_cfg)
stderr_log_path = _parse_server_stderr_log_path(log_cfg)
_ensure_log_dir(stdout_log_path)
_ensure_log_dir(stderr_log_path)
with (
open(stdout_log_path, "a", encoding="utf-8") as stdout_log_file,
open(stderr_log_path, "a", encoding="utf-8") as stderr_log_file,
):
stdout_log_file.write(f"\n--- managed parse-server stdout: {marker} ---\n")
stderr_log_file.write(f"\n--- managed parse-server stderr: {marker} ---\n")
stdout_log_file.flush()
stderr_log_file.flush()
yield stdout_log_file, stderr_log_file
def start_managed_parse_server(
*,
tier: ManagedServerTier,
managed_cfg: ManagedParseServerConfig,
log_cfg: LogConfig | None,
marker: str,
) -> tuple[subprocess.Popen, str, ManagedProcessControl]:
"""创建托管子进程,监听线程或子进程启动失败时回收已创建的控制通道。"""
port = select_available_managed_port(
managed_cfg.host,
managed_cfg.port,
strict_port=managed_cfg.strict_port,
port_probe_count=managed_cfg.port_probe_count,
)
managed_url = managed_parse_server_url(managed_cfg.host, port)
cmd = [
sys.executable,
"-m",
"mineru.parser.api_server",
*api_server_args_for_tier(tier, host=managed_cfg.host, port=port),
]
env = utf8_subprocess_env()
control = ManagedProcessControl.create()
env.update(control.child_env())
logger.info("Starting managed parse-server (%s): %s", marker, " ".join(cmd))
try:
control.start_accepting()
with open_managed_parse_server_logs(marker=marker, log_cfg=log_cfg) as (
stdout_log_file,
stderr_log_file,
):
proc = subprocess.Popen(cmd, stdin=subprocess.DEVNULL, stdout=stdout_log_file, stderr=stderr_log_file, env=env)
except Exception:
control.close()
raise
return proc, managed_url, control
def stop_managed_parse_server(
proc: subprocess.Popen | None,
*,
control: ManagedProcessControl | None = None,
timeout_sec: int,
reason: str,
startup_in_progress: bool = False,
) -> None:
"""在统一预算内关闭托管进程,最终仍未退出时抛错以保留调用方的进程归属。"""
if proc is None:
if control is not None:
control.close()
return
if proc.poll() is not None:
if control is not None:
control.close()
return
pid = proc.pid
total_timeout = max(float(timeout_sec), 0.0)
deadline = time.monotonic() + total_timeout
graceful_cap = min(2.0, total_timeout * 0.2) if startup_in_progress else total_timeout * 0.5
shutdown_sent = control.request_shutdown(graceful_cap) if control is not None else False
if shutdown_sent:
try:
proc.wait(timeout=min(graceful_cap, max(deadline - time.monotonic(), 0.0)))
logger.info("Managed parse-server stopped after control shutdown (PID %d, reason=%s)", pid, reason)
if control is not None:
control.close()
return
except subprocess.TimeoutExpired:
logger.warning(
"Managed parse-server did not stop after control shutdown (PID %d, reason=%s), terminating",
pid,
reason,
)
else:
logger.warning("Managed parse-server control channel unavailable (PID %d, reason=%s), terminating", pid, reason)
try:
proc.terminate()
terminate_cap = total_timeout * 0.3
proc.wait(timeout=min(terminate_cap, max(deadline - time.monotonic(), 0.0)))
logger.info("Managed parse-server terminated (PID %d, reason=%s)", pid, reason)
if control is not None:
control.close()
return
except subprocess.TimeoutExpired:
logger.warning("Managed parse-server did not terminate within timeout (PID %d, reason=%s), killing", pid, reason)
except Exception as exc:
logger.warning("Failed to terminate managed parse-server (PID %d, reason=%s): %s", pid, reason, exc)
try:
proc.kill()
proc.wait(timeout=max(deadline - time.monotonic(), 0.0))
logger.info("Managed parse-server killed (PID %d, reason=%s)", pid, reason)
except Exception as exc:
logger.error("Failed to kill managed parse-server (PID %d, reason=%s): %s", pid, reason, exc)
if proc.poll() is None:
raise RuntimeError(f"Managed parse-server did not stop (PID {pid}, reason={reason})") from exc
finally:
if control is not None:
control.close()
def _managed_parse_server_needs_restart(
health: ParseServerHealth,
*,
now: float,
startup_timeout_sec: int,
) -> bool:
if health.local.probe.error_code in _NON_RETRYABLE_MODEL_PRELOAD_ERRORS:
return False
proc = health.managed_proc
if health.local_starting and proc is not None and proc.poll() is None:
return now - health.local_started_at >= startup_timeout_sec
return True
class ParseServerHealthCheck:
"""Periodically probes parse-server health via GET /v1/tiers."""
def __init__(
self,
config_svc: "ConfigService",
*,
interval_sec: int,
probe_timeout_sec: int,
startup_grace_sec: int,
stop_timeout_sec: int,
health: ParseServerHealth,
startup_timeout_sec: int,
managed_parse_server: ManagedParseServerConfig,
log_cfg: LogConfig,
) -> None:
"""绑定应用健康状态与启动配置,不回落到进程默认配置。"""
self.health = health
self.config_svc = config_svc
self.interval_sec = interval_sec
self.probe_timeout_sec = probe_timeout_sec
self.startup_grace_sec = startup_grace_sec
self.startup_timeout_sec = startup_timeout_sec
self.stop_timeout_sec = stop_timeout_sec
self.managed_parse_server = managed_parse_server
self.log_cfg = log_cfg
self.running = False
self._wakeup_event = asyncio.Event()
def wakeup(self) -> None:
"""配置变更后立即唤醒健康检查,不等待正常探测间隔。"""
self._wakeup_event.set()
async def run(self) -> None:
self.running = True
health = self.health
consecutive_local_failures = 0
while self.running:
self._wakeup_event.clear()
config_generation = health.local_generation
# refresh config on each cycle (hot-reload)
mode = (await self.config_svc.get("parse_server.local.mode")) or "disabled"
desired_managed_tier = await get_managed_parse_server_tier(self.config_svc)
self_hosted_url = await self.config_svc.get("parse_server.local.self_hosted_url")
# 读取配置期间若发生切换,重新读取,避免用旧快照覆盖新的目标档位。
if config_generation != health.local_generation:
continue
if mode == "managed" and (health.local_mode != mode or health.managed_tier != desired_managed_tier):
health.restart_count = 0
health.managed_restart_exhausted = False
health.local_mode = mode
health.managed_tier = desired_managed_tier
health.self_hosted_url = self_hosted_url if self_hosted_url else None
# 档位切换先于探测和远程健康检查,旧服务不能在重启前重新变为可用。
tier_changed = False
if health.local_mode == "managed" and not health.managed_restart_exhausted:
tier_changed = await self._try_restart_managed_for_tier_change(health, desired_managed_tier)
elif health.local_mode != "managed" and (health.managed_proc is not None or health.managed_control is not None):
try:
self._stop_managed_process(health, reason="local mode changed")
except RuntimeError:
logger.exception("Failed to stop managed parse-server after local mode changed")
# probe local
probed_local = False
probe_generation = health.local_generation
if health.local_mode == "managed" and health.managed_restart_exhausted:
self._exhaust_managed_restarts(health)
elif health.local_mode != "disabled":
url = self._local_url(health)
if url:
probed_local = True
api_key = None
if health.local_mode == "self_hosted":
api_key = (await self.config_svc.get("parse_server.local.self_hosted_api_key")) or None
probe = await self._probe(url, api_key=api_key)
if probe_generation != health.local_generation:
continue
if health.local_mode == "managed" and probe.healthy:
if health.running_managed_tier != health.managed_tier or health.managed_tier not in probe.tiers:
probe = ProbeResult(
error_code="parse_server_unavailable",
error_msg=f"Managed parse-server is not ready for tier '{health.managed_tier}'.",
)
now_ms = int(time.time() * 1000)
health.local.url = url
health.local.probe = probe
health.local.last_probe_at = now_ms
if probe.healthy:
health.local.last_success_at = now_ms
health.local_starting = False
health.managed_tier_transition = False
# MAX_RESTART_ATTEMPTS bounds consecutive failed recoveries; a healthy probe restores the budget.
health.restart_count = 0
else:
health.local.last_failure_at = now_ms
if probe.error_code != "parse_server_unavailable":
health.local_starting = False
health.managed_tier_transition = False
else:
health.local = ProbeState()
else:
health.local = ProbeState()
# probe remote
remote_url = cast(str, await self.config_svc.get("parse_server.remote.url"))
remote_api_key = (await resolve_remote_api_key(self.config_svc)).value
probe = await self._probe(remote_url, api_key=remote_api_key)
now_ms = int(time.time() * 1000)
health.remote.url = remote_url
health.remote.probe = probe
health.remote.last_probe_at = now_ms
if probe.healthy:
health.remote.last_success_at = now_ms
else:
health.remote.last_failure_at = now_ms
if probe_generation != health.local_generation:
continue
if health.local_mode == "managed" and not health.managed_restart_exhausted:
proc = health.managed_proc
now = asyncio.get_event_loop().time()
startup_elapsed_sec = now - health.local_started_at
if (
not tier_changed
and health.local_starting
and proc is not None
and proc.poll() is None
and startup_elapsed_sec >= self.startup_grace_sec
and startup_elapsed_sec < self.startup_timeout_sec
):
logger.info(
"Managed parse-server is still preparing startup tier %s (PID %d)",
health.running_managed_tier,
proc.pid,
)
if (
not tier_changed
and not health.local.probe.healthy
and _managed_parse_server_needs_restart(
health,
now=now,
startup_timeout_sec=self.startup_timeout_sec,
)
):
if health.local_starting and proc is not None and proc.poll() is None:
logger.warning(
"Managed parse-server startup exceeded %ds for tier %s (PID %d), attempting restart",
self.startup_timeout_sec,
health.running_managed_tier,
proc.pid,
)
else:
logger.warning("Managed parse-server is unhealthy, attempting restart")
await self._try_restart_managed(health)
# A failed local probe shortens the wait (5/10/20s, capped at the
# normal interval) so a server that becomes ready during startup is
# detected quickly; any success or non-probed cycle restores it.
if probed_local and not health.local.probe.healthy:
consecutive_local_failures += 1
else:
consecutive_local_failures = 0
if self.running:
try:
await asyncio.wait_for(
self._wakeup_event.wait(), timeout=self._next_interval_sec(consecutive_local_failures)
)
except asyncio.TimeoutError:
pass
def _next_interval_sec(self, consecutive_local_failures: int) -> int:
"""下一次探测的等待秒数;每多失败一次多等 5 秒,封顶正常间隔。"""
if consecutive_local_failures <= 0:
return self.interval_sec
return min(consecutive_local_failures * _FAILURE_BACKOFF_STEP_SEC, self.interval_sec)
async def _try_restart_managed_for_tier_change(
self,
health: ParseServerHealth,
desired_managed_tier: ManagedServerTier,
) -> bool:
"""目标档位变化时串行替换托管进程,熔断恢复由配置入口显式触发。"""
if health.managed_restart_exhausted:
return False
running_managed_tier = health.running_managed_tier
if running_managed_tier is None or running_managed_tier == desired_managed_tier:
return False
logger.info(
"Managed parse-server tier changed from %s to %s, restarting",
running_managed_tier,
desired_managed_tier,
)
await self._try_restart_managed(
health,
reason="tier-change",
marker=f"tier change {running_managed_tier}->{desired_managed_tier}",
count_restart=False,
)
return True
async def _try_restart_managed(
self,
health: ParseServerHealth,
*,
reason: str = "restart",
marker: str | None = None,
count_restart: bool = True,
) -> None:
"""先确认旧进程已退出再启动新进程,连续失败达到预算后进入熔断。"""
if health.managed_restart_exhausted or (count_restart and health.restart_count >= MAX_RESTART_ATTEMPTS):
self._exhaust_managed_restarts(health)
return
config_generation = health.local_generation
managed_tier = await get_managed_parse_server_tier(self.config_svc)
if config_generation != health.local_generation:
return
if count_restart:
health.restart_count += 1
health.begin_managed_tier_transition(managed_tier)
try:
self._stop_managed_process(health, reason=reason)
proc, managed_url, control = start_managed_parse_server(
tier=managed_tier,
managed_cfg=self.managed_parse_server,
log_cfg=self.log_cfg,
marker=marker or f"restart attempt {health.restart_count}",
)
health.managed_url = managed_url
logger.info("Managed parse-server restarted (PID %d, tier=%s)", proc.pid, managed_tier)
health.managed_proc = proc
health.managed_control = control
health.running_managed_tier = managed_tier
health.local_starting = True
health.local_started_at = asyncio.get_event_loop().time()
except Exception as exc:
logger.error(f"Failed to restart managed parse-server: {exc}")
# 配置切换成功不消耗恢复预算,但切换失败仍须受连续失败上限约束。
if not count_restart:
health.restart_count += 1
if health.restart_count >= MAX_RESTART_ATTEMPTS:
self._exhaust_managed_restarts(health)
def _stop_managed_process(self, health: ParseServerHealth, *, reason: str) -> None:
"""成功回收后释放进程归属,失败时保留引用供下一轮或应用退出重试。"""
stop_managed_parse_server(
health.managed_proc,
control=health.managed_control,
timeout_sec=self.stop_timeout_sec,
reason=reason,
startup_in_progress=health.local_starting,
)
health.managed_proc = None
health.managed_control = None
health.running_managed_tier = None
health.local_starting = False
health.local_started_at = 0.0
def _exhaust_managed_restarts(self, health: ParseServerHealth) -> None:
"""锁定失败恢复并回收残留进程,后续轮次只重试未完成的回收。"""
if not health.managed_restart_exhausted:
logger.error("Managed parse-server failed %d restarts, disabling automatic restarts", MAX_RESTART_ATTEMPTS)
health.managed_restart_exhausted = True
health.local_generation += 1
health.local.probe = ProbeResult(
error_code="parse_server_unavailable",
error_msg=(
f"Managed parse-server failed {MAX_RESTART_ATTEMPTS} restarts. "
"Set managed mode or tier again, or restart MinerU server to retry."
),
)
try:
if health.managed_proc is not None or health.managed_control is not None:
self._stop_managed_process(health, reason="restart budget exhausted")
except RuntimeError:
logger.exception("Failed to stop managed parse-server after restart budget exhausted")
health.local_starting = False
health.managed_tier_transition = False
async def _probe(self, base_url: str, *, api_key: str | None = None) -> ProbeResult:
headers = {"Authorization": f"Bearer {api_key}"} if api_key else None
try:
async with httpx.AsyncClient(
timeout=self.probe_timeout_sec,
trust_env=should_trust_env_for_url(base_url),
) as client:
resp = await client.get(f"{base_url}/v1/tiers", headers=headers)
if resp.status_code == 200:
data = resp.json()
tiers: list[Tier] = []
for t in data.get("data", []):
tier_id = t.get("id")
if tier_id in TIERS:
tiers.append(cast(Tier, tier_id))
return ProbeResult(healthy=True, tiers=tiers)
error_code, error_msg = _probe_error(resp)
return ProbeResult(error_code=error_code, error_msg=error_msg)
except httpx.TimeoutException as exc:
return ProbeResult(error_code="parse_server_unavailable", error_msg=str(exc) or "Parse-server probe timed out.")
except httpx.TransportError as exc:
return ProbeResult(
error_code="parse_server_unavailable",
error_msg=str(exc) or "Parse-server probe transport failed.",
)
except Exception as exc:
return ProbeResult(error_code="parse_server_unavailable", error_msg=str(exc) or "Parse-server probe failed.")
@staticmethod
def _local_url(health: ParseServerHealth) -> str | None:
if health.local_mode == "managed":
return health.managed_url
if health.local_mode == "self_hosted":
return health.self_hosted_url
return None
async def stop(self) -> None:
"""结束健康检查并确认回收当前应用拥有的托管进程。"""
self.running = False
self.wakeup()
health = self.health
self._stop_managed_process(health, reason="health check stop")
def _probe_error(resp: httpx.Response) -> tuple[str, str]:
data: dict[str, object] = {}
try:
loaded = resp.json()
if isinstance(loaded, dict):
data = loaded
except Exception:
data = {}
error = data.get("error")
if isinstance(error, dict):
code = str(error.get("code") or "http_error")
message = str(error.get("message") or error)
return code, message
remote_message = _remote_auth_message(data)
if resp.status_code == 401 or remote_message is not None:
return "invalid_api_key", remote_message or "API key invalid or remote authentication failed."
text = resp.text[:500]
return "http_error", f"HTTP {resp.status_code}: {text}"
def _remote_auth_message(data: dict[str, object]) -> str | None:
msg_code = data.get("msgCode")
msg = data.get("msg")
if msg_code == "A0202":
return str(msg or "user authenticate failed")
if isinstance(msg, str) and "authenticate failed" in msg.lower():
return msg
return None