DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.
Python ai-agentsai-tutorclawbotcli-tooldeepresearchinteractive-learninglearningmulti-agent-systemsrag
fix(mineru): own CLI workers and avoid log-silence termination
yepyhund48b6278d3ad
diff --git a/deeptutor/services/parsing/engines/mineru/_process.py b/deeptutor/services/parsing/engines/mineru/_process.pynew file mode 100644index 000000000..db2a4ff40--- /dev/null+++ b/deeptutor/services/parsing/engines/mineru/_process.py@@ -0,0 +1,76 @@+"""Own a MinerU CLI tree; settle it before publishing parser output."""++from contextlib import contextmanager+import os+from pathlib import Path+import signal+import subprocess+import sys+import time+++@contextmanager+def cli_process(cmd: list[str], log_path: Path, env: dict[str, str]):+ """Run a CLI with attempt-owned descendants and settle them on every exit."""+ process = None+ job = None+ assigned = False+ with log_path.open("wb") as output:+ try:+ kwargs = {"stdout": output, "stderr": subprocess.STDOUT, "shell": False, "env": env}+ if sys.platform == "win32":+ import psutil+ import win32api+ import win32con+ import win32job+ import win32process++ job = win32job.CreateJobObject(None, "")+ limits = win32job.QueryInformationJobObject(+ job, win32job.JobObjectExtendedLimitInformation+ )+ limits["BasicLimitInformation"]["LimitFlags"] |= (+ win32job.JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE+ )+ win32job.SetInformationJobObject(+ job, win32job.JobObjectExtendedLimitInformation, limits+ )+ # Attach before execution: no child can escape an assignment race.+ kwargs["creationflags"] = win32con.CREATE_SUSPENDED | subprocess.CREATE_NO_WINDOW+ process = subprocess.Popen(cmd, **kwargs) # nosec B603: fixed argv, shell=False+ win32job.AssignProcessToJobObject(job, int(process._handle))+ assigned = True+ threads = psutil.Process(process.pid).threads()+ if len(threads) != 1:+ raise RuntimeError("Cannot identify the suspended MinerU main thread")+ thread = win32api.OpenThread(win32con.THREAD_SUSPEND_RESUME, False, threads[0].id)+ try:+ win32process.ResumeThread(thread)+ finally:+ thread.Close()+ else:+ process = subprocess.Popen(cmd, start_new_session=True, **kwargs) # nosec B603+ yield process+ finally:+ if sys.platform == "win32" and job is not None:+ try:+ if process is not None and not assigned:+ process.kill() # Still suspended; cannot have spawned a child.+ win32job.TerminateJobObject(job, 1)+ deadline = time.monotonic() + 5+ while win32job.QueryInformationJobObject(+ job, win32job.JobObjectBasicAccountingInformation+ )["ActiveProcesses"]:+ if time.monotonic() >= deadline:+ raise RuntimeError("MinerU process-tree cleanup did not settle")+ time.sleep(0.05)+ finally:+ job.Close()+ elif process is not None:+ for sig in (signal.SIGTERM, signal.SIGKILL):+ try:+ os.killpg(process.pid, sig)+ except ProcessLookupError:+ break+ if process is not None:+ process.wait(timeout=5)diff --git a/deeptutor/services/parsing/engines/mineru/local.py b/deeptutor/services/parsing/engines/mineru/local.pyindex f3861b535..d502260d8 100644--- a/deeptutor/services/parsing/engines/mineru/local.py+++ b/deeptutor/services/parsing/engines/mineru/local.py@@ -6,19 +6,20 @@ from collections import dequefrom collections.abc import Callablefrom dataclasses import dataclassfrom enum import StrEnum+import loggingimport osfrom pathlib import Pathimport shutilimport subprocessimport sysimport tempfile-import threadingimport timefrom uuid import uuid4from deeptutor.services.file_io import atomic_write_jsonfrom deeptutor.services.parsing.cache import load_ir+from ._process import cli_processfrom .formats import MINERU_SUPPORTED_FORMATS# Minimum seconds between on_output callbacks. MinerU's CLI emits tqdm-style@@ -207,88 +208,105 @@ def parse_document_with_mineru_result(temp_output.mkdir()state_path = attempt / "state.json"atomic_write_json(state_path, {"source": source_file.name, "state": "running"})- process = None- process_finished = False- watchdog_stop = threading.Event()- watchdog = Noneinterrupted: list[LocalParseReason] = []+ interruption_detail = ""result = LocalParseResult.failure(LocalParseReason.EXCEPTION, "parse interrupted")try:cmd = [mineru_cmd, "-p", str(source_file), "-o", str(temp_output)]print(f"🔧 Executing command: {' '.join(cmd)}")- # Stream combined stdout/stderr line by line. text=True enables- # universal newlines, so tqdm's \r-rewritten progress bars arrive as- # individual lines rather than one giant buffered blob at exit.- process = subprocess.Popen( # nosec B603 — fixed argv, shell=False- cmd,- stdout=subprocess.PIPE,- stderr=subprocess.STDOUT,- text=True,- encoding="utf-8",- errors="replace",- shell=False,- env={**os.environ, **extra_env} if extra_env else None,- )tail: deque[str] = deque(maxlen=40)last_emit = 0.0from deeptutor.knowledge.indexing_run import IndexingCancelled, current_runrun = current_run()- activity = [time.monotonic()]- if run is not None:- started = activity[0]-- def watch():- while not watchdog_stop.wait(0.25):- try:- run.check()- except IndexingCancelled:- interrupted.append(LocalParseReason.CANCELLED)- except OSError:- interrupted.append(LocalParseReason.EXCEPTION)- if (- time.monotonic() - activity[0] > _LOCAL_PARSE_IDLE_TIMEOUT_SECONDS- or time.monotonic() - started > _LOCAL_PARSE_TIMEOUT_SECONDS- ):- interrupted.append(LocalParseReason.TIMEOUT)- if interrupted:+ started = activity = time.monotonic()+ quiet_reported = False+ total_timeout = _LOCAL_PARSE_TIMEOUT_SECONDS+ try:+ configured_timeout = int(+ (extra_env or {}).get(+ "MINERU_TASK_RESULT_TIMEOUT_SECONDS",+ os.environ.get("MINERU_TASK_RESULT_TIMEOUT_SECONDS", str(total_timeout)),+ )+ )+ if configured_timeout > 0:+ total_timeout = configured_timeout+ except (TypeError, ValueError):+ pass+ env = {**os.environ, **(extra_env or {}), "PYTHONUTF8": "1"}+ log_path = attempt / "cli.log"+ # File reads do not block on a pipe inherited by a surviving worker.+ # Settle the owned process tree before validating/publishing artifacts.+ with cli_process(cmd, log_path, env) as process:+ with log_path.open("r", encoding="utf-8", errors="replace") as output:+ while True:+ if run is not None:try:- process.terminate()- process.wait(timeout=3)- except subprocess.TimeoutExpired:- process.kill()- except OSError:- pass- return-- watchdog = threading.Thread(target=watch, daemon=True)- watchdog.start()- assert process.stdout is not None- for raw_line in process.stdout:- line = raw_line.strip()- if not line:- continue- activity[0] = time.monotonic()- tail.append(line)- if on_output is not None:- now = time.monotonic()- if now - last_emit >= _ON_OUTPUT_MIN_INTERVAL:- last_emit = now- try:- on_output(line[:300])- except Exception:- # A broken callback must not kill the parse; stop- # reporting and keep going.- on_output = None- returncode = process.wait()- process_finished = True+ run.check()+ except IndexingCancelled:+ interrupted.append(LocalParseReason.CANCELLED)+ interruption_detail = "Task cancellation requested."+ break+ except OSError as exc:+ interrupted.append(LocalParseReason.EXCEPTION)+ interruption_detail = f"Indexing state check failed: {exc}"+ break+ for _ in range(32):+ raw_line = output.readline(4096)+ if not raw_line:+ break+ line = raw_line.strip()+ if not line:+ continue+ activity = time.monotonic()+ quiet_reported = False+ tail.append(line)+ if (+ on_output is not None+ and activity - last_emit >= _ON_OUTPUT_MIN_INTERVAL+ ):+ last_emit = activity+ try:+ on_output(line[:300])+ except Exception:+ on_output = None+ returncode = process.poll()+ if returncode is not None:+ break+ now = time.monotonic()+ if now - started >= total_timeout:+ interrupted.append(LocalParseReason.TIMEOUT)+ interruption_detail = (+ f"Total parse runtime exceeded {total_timeout:g} seconds."+ )+ break+ if now - activity >= _LOCAL_PARSE_IDLE_TIMEOUT_SECONDS and not quiet_reported:+ logging.getLogger(__name__).warning(+ "MinerU output is quiet; progress is unknown, not a proven stall. "+ "The %s-second total deadline and task cancellation remain active.",+ total_timeout,+ )+ quiet_reported = True+ time.sleep(0.1)++ # Cleanup has settled the owned tree. Read only a bounded diagnostic+ # tail now; draining a still-writing descendant before cleanup could+ # otherwise postpone cancellation/exit indefinitely.+ with log_path.open("rb") as output:+ output.seek(0, os.SEEK_END)+ start = max(0, output.tell() - 16384)+ output.seek(start)+ lines = output.read(16384).decode("utf-8", errors="replace").splitlines()+ if start:+ lines = lines[1:]+ tail.extend(line.strip()[:4096] for line in lines if line.strip())if interrupted:result = LocalParseResult.failure(interrupted[0],- "Cancellation or parse time limit reached; completed checkpoints remain reusable.",+ interruption_detail + " Completed checkpoints remain reusable.",)return resultif returncode != 0:@@ -356,17 +374,6 @@ def parse_document_with_mineru_result()return resultfinally:- watchdog_stop.set()- if watchdog is not None:- watchdog.join(timeout=4)- # Stop an interrupted child before another attempt can publish output.- if process is not None and not process_finished:- try:- process.terminate()- process.wait(timeout=3)- except subprocess.TimeoutExpired:- process.kill()- process.wait()if result.ok:shutil.rmtree(attempt, ignore_errors=True)else:diff --git a/pyproject.toml b/pyproject.tomlindex 0f5bc7c5e..9723f6d42 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -91,6 +91,7 @@ dependencies = [# deeptutor.runtime.memory_probe imports it lazily and falls back to /proc,# so a missing wheel costs the settings strip, not the app."psutil>=5.9.0,<8.0.0",+ "pywin32; sys_platform == 'win32'", # MinerU Job Object ownership][project.scripts]diff --git a/requirements/cli.txt b/requirements/cli.txtindex 092bdaa1a..7c2b4231d 100644--- a/requirements/cli.txt+++ b/requirements/cli.txt@@ -49,6 +49,8 @@ llama-index-vector-stores-faiss>=0.4.0,<1.0.0faiss-cpu>=1.8.0,<2.0.0; python_version < "3.14"faiss-cpu>=1.12.0,<2.0.0; python_version >= "3.14"PyMuPDF>=1.26.0+psutil>=5.9.0,<8.0.0+pywin32; sys_platform == "win32"Pillow>=10.0.0numpy>=1.24.0,<3.0.0arxiv>=2.0.0diff --git a/tests/services/parsing/test_mineru_local_failures.py b/tests/services/parsing/test_mineru_local_failures.pyindex b98d59180..df745f3f2 100644--- a/tests/services/parsing/test_mineru_local_failures.py+++ b/tests/services/parsing/test_mineru_local_failures.py@@ -8,6 +8,7 @@ unchanged bool contract of the legacy helpers.from __future__ import annotations+from contextlib import contextmanagerfrom pathlib import Pathimport subprocess@@ -51,10 +52,16 @@ def _install_fake_popen(path.write_text("# parsed", encoding="utf-8")self.stdout = iter(lines)- def wait(self) -> int:+ def poll(self) -> int:return returncode- monkeypatch.setattr(mineru_local.subprocess, "Popen", FakeProcess)+ @contextmanager+ def fake_cli_process(cmd, log_path, env):+ process = FakeProcess(cmd)+ log_path.write_text("\n".join(lines), encoding="utf-8")+ yield process++ monkeypatch.setattr(mineru_local, "cli_process", fake_cli_process)# ---------------------------------------------------------------------------@@ -228,10 +235,8 @@ def test_interrupted_attempt_stops_child_and_retains_partial_output(def __init__(self, cmd, **kwargs):target = Path(cmd[cmd.index("-o") + 1])(target / "partial.md").write_text("partial", encoding="utf-8")- self.stdout = self.lines()- def lines(self):- yield "parsing page 2"+ def poll(self):raise KeyboardInterruptdef terminate(self):@@ -243,12 +248,18 @@ def test_interrupted_attempt_stops_child_and_retains_partial_output(child = None- def start(*args, **kwargs):+ @contextmanager+ def start(cmd, log_path, env):nonlocal child- child = InterruptedProcess(*args, **kwargs)- return child-- monkeypatch.setattr(mineru_local.subprocess, "Popen", start)+ child = InterruptedProcess(cmd)+ log_path.write_text("parsing page 2\n", encoding="utf-8")+ try:+ yield child+ finally:+ child.terminate()+ child.wait()++ monkeypatch.setattr(mineru_local, "cli_process", start)with pytest.raises(KeyboardInterrupt):parse_document_with_mineru_result(pdf, tmp_path / "out", cli_command="mineru")assert child.stoppeddiff --git a/tests/services/parsing/test_mineru_process_lifecycle.py b/tests/services/parsing/test_mineru_process_lifecycle.pynew file mode 100644index 000000000..f8777f686--- /dev/null+++ b/tests/services/parsing/test_mineru_process_lifecycle.py@@ -0,0 +1,97 @@+"""Real subprocess regression checks without MinerU/model downloads."""++from pathlib import Path+import subprocess+import sys+import time+from types import SimpleNamespace++import psutil+import pytest++from deeptutor.knowledge import indexing_run+from deeptutor.services.parsing.engines.mineru import local+++def run_fixture(tmp_path, monkeypatch, code, run=None, idle=0.05, timeout=30):+ source = tmp_path / "lesson.pdf"+ source.write_bytes(b"controlled fixture")+ popen = subprocess.Popen++ def spawn(cmd, **kwargs):+ output = cmd[cmd.index("-o") + 1]+ return popen([sys.executable, "-c", code, output], **kwargs)++ monkeypatch.setattr(local.subprocess, "Popen", spawn)+ monkeypatch.setattr(+ indexing_run, "current_run", lambda: run or SimpleNamespace(check=lambda: None)+ )+ monkeypatch.setattr(local, "_LOCAL_PARSE_IDLE_TIMEOUT_SECONDS", idle)+ monkeypatch.setattr(local, "_LOCAL_PARSE_TIMEOUT_SECONDS", timeout)+ monkeypatch.delenv("MINERU_TASK_RESULT_TIMEOUT_SECONDS", raising=False)+ started = time.monotonic()+ result = local.parse_document_with_mineru_result(+ source,+ tmp_path / "out",+ cli_command=sys.executable,+ )+ return result, time.monotonic() - started+++def test_quiet_inference_is_not_killed_for_log_silence(tmp_path, monkeypatch):+ code = (+ "import sys,time,pathlib; time.sleep(0.7); "+ "p=pathlib.Path(sys.argv[1]); (p/'lesson.md').write_text('# parsed',encoding='utf-8')"+ )+ result, elapsed = run_fixture(tmp_path, monkeypatch, code)+ assert result.ok, result+ assert elapsed < 2+++def test_parent_exit_does_not_wait_for_descendant_stdout(tmp_path, monkeypatch):+ # A finite child keeps the red test bounded even with the old broken reader.+ code = (+ "import sys,subprocess,pathlib; p=pathlib.Path(sys.argv[1]); "+ "c=subprocess.Popen([sys.executable,'-c','import time; time.sleep(2)'],"+ "stdout=sys.stdout,stderr=sys.stderr); "+ "(p/'child.pid').write_text(str(c.pid)); "+ "(p/'lesson.md').write_text('# parsed',encoding='utf-8')"+ )+ unrelated = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(10)"])+ try:+ result, elapsed = run_fixture(tmp_path, monkeypatch, code, idle=10)+ assert result.ok, result+ assert elapsed < 1.5, "Waiting for inherited stdout instead of parent exit"+ child = int((tmp_path / "out" / "lesson" / "child.pid").read_text())+ assert not psutil.pid_exists(child), "Owned child escaped cleanup"+ assert unrelated.poll() is None, "Cleanup touched an unrelated process"+ finally:+ unrelated.terminate()+ unrelated.wait(timeout=5)++[email protected]("cancelled", [False, True])+def test_deadline_and_cancellation_settle_the_owned_tree(tmp_path, monkeypatch, cancelled):+ code = (+ "import sys,subprocess,pathlib,time; p=pathlib.Path(sys.argv[1]); "+ "c=subprocess.Popen([sys.executable,'-c','import time; time.sleep(10)']); "+ "(p/'child.pid').write_text(str(c.pid)); time.sleep(10)"+ )++ def check():+ if cancelled and list((tmp_path / "out").glob(".mineru-attempt-*/output/child.pid")):+ raise indexing_run.IndexingCancelled("controlled cancellation")++ result, elapsed = run_fixture(+ tmp_path,+ monkeypatch,+ code,+ run=SimpleNamespace(check=check),+ timeout=0.8,+ )+ assert result.reason is (+ local.LocalParseReason.CANCELLED if cancelled else local.LocalParseReason.TIMEOUT+ )+ assert elapsed < 2+ (child_file,) = (tmp_path / "out").glob(".mineru-attempt-*/output/child.pid")+ assert not psutil.pid_exists(int(child_file.read_text()))