Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix(mineru): own CLI workers and avoid log-silence termination

yepyhund48b6278d3ad
diff --git a/deeptutor/services/parsing/engines/mineru/_process.py b/deeptutor/services/parsing/engines/mineru/_process.py
new file mode 100644
index 000000000..db2a4ff40
--- /dev/null
+++ b/deeptutor/services/parsing/engines/mineru/_process.py
@@ -0,0 +1,76 @@
+"""Own a MinerU CLI tree; settle it before publishing parser output."""
+
+from contextlib import contextmanager
+import os
+from pathlib import Path
+import signal
+import subprocess
+import sys
+import time
+
+
+@contextmanager
+def cli_process(cmd: list[str], log_path: Path, env: dict[str, str]):
+ """Run a CLI with attempt-owned descendants and settle them on every exit."""
+ process = None
+ job = None
+ assigned = False
+ with log_path.open("wb") as output:
+ try:
+ kwargs = {"stdout": output, "stderr": subprocess.STDOUT, "shell": False, "env": env}
+ if sys.platform == "win32":
+ import psutil
+ import win32api
+ import win32con
+ import win32job
+ import win32process
+
+ job = win32job.CreateJobObject(None, "")
+ limits = win32job.QueryInformationJobObject(
+ job, win32job.JobObjectExtendedLimitInformation
+ )
+ limits["BasicLimitInformation"]["LimitFlags"] |= (
+ win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE
+ )
+ win32job.SetInformationJobObject(
+ job, win32job.JobObjectExtendedLimitInformation, limits
+ )
+ # Attach before execution: no child can escape an assignment race.
+ kwargs["creationflags"] = win32con.CREATE_SUSPENDED | subprocess.CREATE_NO_WINDOW
+ process = subprocess.Popen(cmd, **kwargs) # nosec B603: fixed argv, shell=False
+ win32job.AssignProcessToJobObject(job, int(process._handle))
+ assigned = True
+ threads = psutil.Process(process.pid).threads()
+ if len(threads) != 1:
+ raise RuntimeError("Cannot identify the suspended MinerU main thread")
+ thread = win32api.OpenThread(win32con.THREAD_SUSPEND_RESUME, False, threads[0].id)
+ try:
+ win32process.ResumeThread(thread)
+ finally:
+ thread.Close()
+ else:
+ process = subprocess.Popen(cmd, start_new_session=True, **kwargs) # nosec B603
+ yield process
+ finally:
+ if sys.platform == "win32" and job is not None:
+ try:
+ if process is not None and not assigned:
+ process.kill() # Still suspended; cannot have spawned a child.
+ win32job.TerminateJobObject(job, 1)
+ deadline = time.monotonic() + 5
+ while win32job.QueryInformationJobObject(
+ job, win32job.JobObjectBasicAccountingInformation
+ )["ActiveProcesses"]:
+ if time.monotonic() >= deadline:
+ raise RuntimeError("MinerU process-tree cleanup did not settle")
+ time.sleep(0.05)
+ finally:
+ job.Close()
+ elif process is not None:
+ for sig in (signal.SIGTERM, signal.SIGKILL):
+ try:
+ os.killpg(process.pid, sig)
+ except ProcessLookupError:
+ break
+ if process is not None:
+ process.wait(timeout=5)
diff --git a/deeptutor/services/parsing/engines/mineru/local.py b/deeptutor/services/parsing/engines/mineru/local.py
index f3861b535..d502260d8 100644
--- a/deeptutor/services/parsing/engines/mineru/local.py
+++ b/deeptutor/services/parsing/engines/mineru/local.py
@@ -6,19 +6,20 @@ from collections import deque
from collections.abc import Callable
from dataclasses import dataclass
from enum import StrEnum
+import logging
import os
from pathlib import Path
import shutil
import subprocess
import sys
import tempfile
-import threading
import time
from uuid import uuid4
from deeptutor.services.file_io import atomic_write_json
from deeptutor.services.parsing.cache import load_ir
+from ._process import cli_process
from .formats import MINERU_SUPPORTED_FORMATS
# Minimum seconds between on_output callbacks. MinerU's CLI emits tqdm-style
@@ -207,88 +208,105 @@ def parse_document_with_mineru_result(
temp_output.mkdir()
state_path = attempt / "state.json"
atomic_write_json(state_path, {"source": source_file.name, "state": "running"})
- process = None
- process_finished = False
- watchdog_stop = threading.Event()
- watchdog = None
interrupted: list[LocalParseReason] = []
+ interruption_detail = ""
result = LocalParseResult.failure(LocalParseReason.EXCEPTION, "parse interrupted")
try:
cmd = [mineru_cmd, "-p", str(source_file), "-o", str(temp_output)]
print(f"🔧 Executing command: {' '.join(cmd)}")
- # Stream combined stdout/stderr line by line. text=True enables
- # universal newlines, so tqdm's \r-rewritten progress bars arrive as
- # individual lines rather than one giant buffered blob at exit.
- process = subprocess.Popen( # nosec B603 — fixed argv, shell=False
- cmd,
- stdout=subprocess.PIPE,
- stderr=subprocess.STDOUT,
- text=True,
- encoding="utf-8",
- errors="replace",
- shell=False,
- env={**os.environ, **extra_env} if extra_env else None,
- )
tail: deque[str] = deque(maxlen=40)
last_emit = 0.0
from deeptutor.knowledge.indexing_run import IndexingCancelled, current_run
run = current_run()
- activity = [time.monotonic()]
- if run is not None:
- started = activity[0]
-
- def watch():
- while not watchdog_stop.wait(0.25):
- try:
- run.check()
- except IndexingCancelled:
- interrupted.append(LocalParseReason.CANCELLED)
- except OSError:
- interrupted.append(LocalParseReason.EXCEPTION)
- if (
- time.monotonic() - activity[0] > _LOCAL_PARSE_IDLE_TIMEOUT_SECONDS
- or time.monotonic() - started > _LOCAL_PARSE_TIMEOUT_SECONDS
- ):
- interrupted.append(LocalParseReason.TIMEOUT)
- if interrupted:
+ started = activity = time.monotonic()
+ quiet_reported = False
+ total_timeout = _LOCAL_PARSE_TIMEOUT_SECONDS
+ try:
+ configured_timeout = int(
+ (extra_env or {}).get(
+ "MINERU_TASK_RESULT_TIMEOUT_SECONDS",
+ os.environ.get("MINERU_TASK_RESULT_TIMEOUT_SECONDS", str(total_timeout)),
+ )
+ )
+ if configured_timeout > 0:
+ total_timeout = configured_timeout
+ except (TypeError, ValueError):
+ pass
+ env = {**os.environ, **(extra_env or {}), "PYTHONUTF8": "1"}
+ log_path = attempt / "cli.log"
+ # File reads do not block on a pipe inherited by a surviving worker.
+ # Settle the owned process tree before validating/publishing artifacts.
+ with cli_process(cmd, log_path, env) as process:
+ with log_path.open("r", encoding="utf-8", errors="replace") as output:
+ while True:
+ if run is not None:
try:
- process.terminate()
- process.wait(timeout=3)
- except subprocess.TimeoutExpired:
- process.kill()
- except OSError:
- pass
- return
-
- watchdog = threading.Thread(target=watch, daemon=True)
- watchdog.start()
- assert process.stdout is not None
- for raw_line in process.stdout:
- line = raw_line.strip()
- if not line:
- continue
- activity[0] = time.monotonic()
- tail.append(line)
- if on_output is not None:
- now = time.monotonic()
- if now - last_emit >= _ON_OUTPUT_MIN_INTERVAL:
- last_emit = now
- try:
- on_output(line[:300])
- except Exception:
- # A broken callback must not kill the parse; stop
- # reporting and keep going.
- on_output = None
- returncode = process.wait()
- process_finished = True
+ run.check()
+ except IndexingCancelled:
+ interrupted.append(LocalParseReason.CANCELLED)
+ interruption_detail = "Task cancellation requested."
+ break
+ except OSError as exc:
+ interrupted.append(LocalParseReason.EXCEPTION)
+ interruption_detail = f"Indexing state check failed: {exc}"
+ break
+ for _ in range(32):
+ raw_line = output.readline(4096)
+ if not raw_line:
+ break
+ line = raw_line.strip()
+ if not line:
+ continue
+ activity = time.monotonic()
+ quiet_reported = False
+ tail.append(line)
+ if (
+ on_output is not None
+ and activity - last_emit >= _ON_OUTPUT_MIN_INTERVAL
+ ):
+ last_emit = activity
+ try:
+ on_output(line[:300])
+ except Exception:
+ on_output = None
+ returncode = process.poll()
+ if returncode is not None:
+ break
+ now = time.monotonic()
+ if now - started >= total_timeout:
+ interrupted.append(LocalParseReason.TIMEOUT)
+ interruption_detail = (
+ f"Total parse runtime exceeded {total_timeout:g} seconds."
+ )
+ break
+ if now - activity >= _LOCAL_PARSE_IDLE_TIMEOUT_SECONDS and not quiet_reported:
+ logging.getLogger(__name__).warning(
+ "MinerU output is quiet; progress is unknown, not a proven stall. "
+ "The %s-second total deadline and task cancellation remain active.",
+ total_timeout,
+ )
+ quiet_reported = True
+ time.sleep(0.1)
+
+ # Cleanup has settled the owned tree. Read only a bounded diagnostic
+ # tail now; draining a still-writing descendant before cleanup could
+ # otherwise postpone cancellation/exit indefinitely.
+ with log_path.open("rb") as output:
+ output.seek(0, os.SEEK_END)
+ start = max(0, output.tell() - 16384)
+ output.seek(start)
+ lines = output.read(16384).decode("utf-8", errors="replace").splitlines()
+ if start:
+ lines = lines[1:]
+ tail.extend(line.strip()[:4096] for line in lines if line.strip())
if interrupted:
result = LocalParseResult.failure(
interrupted[0],
- "Cancellation or parse time limit reached; completed checkpoints remain reusable.",
+ interruption_detail + " Completed checkpoints remain reusable.",
)
return result
if returncode != 0:
@@ -356,17 +374,6 @@ def parse_document_with_mineru_result(
)
return result
finally:
- watchdog_stop.set()
- if watchdog is not None:
- watchdog.join(timeout=4)
- # Stop an interrupted child before another attempt can publish output.
- if process is not None and not process_finished:
- try:
- process.terminate()
- process.wait(timeout=3)
- except subprocess.TimeoutExpired:
- process.kill()
- process.wait()
if result.ok:
shutil.rmtree(attempt, ignore_errors=True)
else:
diff --git a/pyproject.toml b/pyproject.toml
index 0f5bc7c5e..9723f6d42 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -91,6 +91,7 @@ dependencies = [
# deeptutor.runtime.memory_probe imports it lazily and falls back to /proc,
# so a missing wheel costs the settings strip, not the app.
"psutil>=5.9.0,<8.0.0",
+ "pywin32; sys_platform == 'win32'", # MinerU Job Object ownership
]
[project.scripts]
diff --git a/requirements/cli.txt b/requirements/cli.txt
index 092bdaa1a..7c2b4231d 100644
--- a/requirements/cli.txt
+++ b/requirements/cli.txt
@@ -49,6 +49,8 @@ llama-index-vector-stores-faiss>=0.4.0,<1.0.0
faiss-cpu>=1.8.0,<2.0.0; python_version < "3.14"
faiss-cpu>=1.12.0,<2.0.0; python_version >= "3.14"
PyMuPDF>=1.26.0
+psutil>=5.9.0,<8.0.0
+pywin32; sys_platform == "win32"
Pillow>=10.0.0
numpy>=1.24.0,<3.0.0
arxiv>=2.0.0
diff --git a/tests/services/parsing/test_mineru_local_failures.py b/tests/services/parsing/test_mineru_local_failures.py
index b98d59180..df745f3f2 100644
--- a/tests/services/parsing/test_mineru_local_failures.py
+++ b/tests/services/parsing/test_mineru_local_failures.py
@@ -8,6 +8,7 @@ unchanged bool contract of the legacy helpers.
from __future__ import annotations
+from contextlib import contextmanager
from pathlib import Path
import subprocess
@@ -51,10 +52,16 @@ def _install_fake_popen(
path.write_text("# parsed", encoding="utf-8")
self.stdout = iter(lines)
- def wait(self) -> int:
+ def poll(self) -> int:
return returncode
- monkeypatch.setattr(mineru_local.subprocess, "Popen", FakeProcess)
+ @contextmanager
+ def fake_cli_process(cmd, log_path, env):
+ process = FakeProcess(cmd)
+ log_path.write_text("\n".join(lines), encoding="utf-8")
+ yield process
+
+ monkeypatch.setattr(mineru_local, "cli_process", fake_cli_process)
# ---------------------------------------------------------------------------
@@ -228,10 +235,8 @@ def test_interrupted_attempt_stops_child_and_retains_partial_output(
def __init__(self, cmd, **kwargs):
target = Path(cmd[cmd.index("-o") + 1])
(target / "partial.md").write_text("partial", encoding="utf-8")
- self.stdout = self.lines()
- def lines(self):
- yield "parsing page 2"
+ def poll(self):
raise KeyboardInterrupt
def terminate(self):
@@ -243,12 +248,18 @@ def test_interrupted_attempt_stops_child_and_retains_partial_output(
child = None
- def start(*args, **kwargs):
+ @contextmanager
+ def start(cmd, log_path, env):
nonlocal child
- child = InterruptedProcess(*args, **kwargs)
- return child
-
- monkeypatch.setattr(mineru_local.subprocess, "Popen", start)
+ child = InterruptedProcess(cmd)
+ log_path.write_text("parsing page 2\n", encoding="utf-8")
+ try:
+ yield child
+ finally:
+ child.terminate()
+ child.wait()
+
+ monkeypatch.setattr(mineru_local, "cli_process", start)
with pytest.raises(KeyboardInterrupt):
parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")
assert child.stopped
diff --git a/tests/services/parsing/test_mineru_process_lifecycle.py b/tests/services/parsing/test_mineru_process_lifecycle.py
new file mode 100644
index 000000000..f8777f686
--- /dev/null
+++ b/tests/services/parsing/test_mineru_process_lifecycle.py
@@ -0,0 +1,97 @@
+"""Real subprocess regression checks without MinerU/model downloads."""
+
+from pathlib import Path
+import subprocess
+import sys
+import time
+from types import SimpleNamespace
+
+import psutil
+import pytest
+
+from deeptutor.knowledge import indexing_run
+from deeptutor.services.parsing.engines.mineru import local
+
+
+def run_fixture(tmp_path, monkeypatch, code, run=None, idle=0.05, timeout=30):
+ source = tmp_path / "lesson.pdf"
+ source.write_bytes(b"controlled fixture")
+ popen = subprocess.Popen
+
+ def spawn(cmd, **kwargs):
+ output = cmd[cmd.index("-o") + 1]
+ return popen([sys.executable, "-c", code, output], **kwargs)
+
+ monkeypatch.setattr(local.subprocess, "Popen", spawn)
+ monkeypatch.setattr(
+ indexing_run, "current_run", lambda: run or SimpleNamespace(check=lambda: None)
+ )
+ monkeypatch.setattr(local, "_LOCAL_PARSE_IDLE_TIMEOUT_SECONDS", idle)
+ monkeypatch.setattr(local, "_LOCAL_PARSE_TIMEOUT_SECONDS", timeout)
+ monkeypatch.delenv("MINERU_TASK_RESULT_TIMEOUT_SECONDS", raising=False)
+ started = time.monotonic()
+ result = local.parse_document_with_mineru_result(
+ source,
+ tmp_path / "out",
+ cli_command=sys.executable,
+ )
+ return result, time.monotonic() - started
+
+
+def test_quiet_inference_is_not_killed_for_log_silence(tmp_path, monkeypatch):
+ code = (
+ "import sys,time,pathlib; time.sleep(0.7); "
+ "p=pathlib.Path(sys.argv[1]); (p/'lesson.md').write_text('# parsed',encoding='utf-8')"
+ )
+ result, elapsed = run_fixture(tmp_path, monkeypatch, code)
+ assert result.ok, result
+ assert elapsed < 2
+
+
+def test_parent_exit_does_not_wait_for_descendant_stdout(tmp_path, monkeypatch):
+ # A finite child keeps the red test bounded even with the old broken reader.
+ code = (
+ "import sys,subprocess,pathlib; p=pathlib.Path(sys.argv[1]); "
+ "c=subprocess.Popen([sys.executable,'-c','import time; time.sleep(2)'],"
+ "stdout=sys.stdout,stderr=sys.stderr); "
+ "(p/'child.pid').write_text(str(c.pid)); "
+ "(p/'lesson.md').write_text('# parsed',encoding='utf-8')"
+ )
+ unrelated = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(10)"])
+ try:
+ result, elapsed = run_fixture(tmp_path, monkeypatch, code, idle=10)
+ assert result.ok, result
+ assert elapsed < 1.5, "Waiting for inherited stdout instead of parent exit"
+ child = int((tmp_path / "out" / "lesson" / "child.pid").read_text())
+ assert not psutil.pid_exists(child), "Owned child escaped cleanup"
+ assert unrelated.poll() is None, "Cleanup touched an unrelated process"
+ finally:
+ unrelated.terminate()
+ unrelated.wait(timeout=5)
+
+
[email protected]("cancelled", [False, True])
+def test_deadline_and_cancellation_settle_the_owned_tree(tmp_path, monkeypatch, cancelled):
+ code = (
+ "import sys,subprocess,pathlib,time; p=pathlib.Path(sys.argv[1]); "
+ "c=subprocess.Popen([sys.executable,'-c','import time; time.sleep(10)']); "
+ "(p/'child.pid').write_text(str(c.pid)); time.sleep(10)"
+ )
+
+ def check():
+ if cancelled and list((tmp_path / "out").glob(".mineru-attempt-*/output/child.pid")):
+ raise indexing_run.IndexingCancelled("controlled cancellation")
+
+ result, elapsed = run_fixture(
+ tmp_path,
+ monkeypatch,
+ code,
+ run=SimpleNamespace(check=check),
+ timeout=0.8,
+ )
+ assert result.reason is (
+ local.LocalParseReason.CANCELLED if cancelled else local.LocalParseReason.TIMEOUT
+ )
+ assert elapsed < 2
+ (child_file,) = (tmp_path / "out").glob(".mineru-attempt-*/output/child.pid")
+ assert not psutil.pid_exists(int(child_file.read_text()))