Open-source LLM knowledge platform: turn raw documents into a queryable RAG, an autonomous reasoning agent, and a self-maintaining Wiki.
Go agentagenticaichatbotdsh-pluginembeddingsevaluationgenerative-aigolangknowledge-basellmmulti-tenantollamaopenaiquestion-answeringragrerankingsemantic-searchvector-searchwiki
fix(docreader): kill timed-out converter process groups (#2990)
YixuanLieu06ae6bcf04e5
diff --git a/docreader/parser/doc_parser.py b/docreader/parser/doc_parser.pyindex 8c0882a3d..013aa3ee7 100644--- a/docreader/parser/doc_parser.py+++ b/docreader/parser/doc_parser.py@@ -1,5 +1,6 @@import loggingimport os+import signalimport subprocessimport timeimport uuid@@ -75,22 +76,93 @@ class SandboxExecutor:if self.proxy:logger.info(f"Using proxy: {self.proxy}")+ is_posix = os.name == "posix"process = subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.PIPE,env=env,+ # On POSIX, start a fresh group so a timed-out command can be+ # terminated together with children it spawned (e.g. soffice.bin).+ start_new_session=is_posix,)+ # On POSIX start_new_session makes the direct child the group leader, so+ # retain this ID even if the direct child exits before cleanup starts.+ process_group_id = process.pid if is_posix else Nonetry:stdout, stderr = process.communicate(timeout=self.default_timeout)return stdout, stderr, process.returncodeexcept subprocess.TimeoutExpired:- process.kill()+ # Terminate the POSIX process group (or the direct child elsewhere),+ # then reap with a bounded wait. Do not drain pipes here:+ # communicate() waits for EOF and can hang if a descendant left+ # the group (setsid) or otherwise kept a PIPE writer open.+ try:+ self._kill_timed_out_command(process, process_group_id)+ finally:+ self._reap_timed_out_process(process)raise RuntimeError(f"Command execution timeout after {self.default_timeout} seconds")+ def _kill_timed_out_command(+ self, process: subprocess.Popen, process_group_id: Optional[int]+ ) -> None:+ if process_group_id is None:+ try:+ process.kill()+ except OSError:+ pass+ return++ self._signal_process_group(process_group_id, signal.SIGTERM)+ grace_period = 0.5+ grace_deadline = time.monotonic() + grace_period+ try:+ process.wait(timeout=grace_period)+ except subprocess.TimeoutExpired:+ pass+ remaining_grace_period = grace_deadline - time.monotonic()+ if remaining_grace_period > 0:+ time.sleep(remaining_grace_period)+ # The direct child can exit after SIGTERM while a descendant+ # ignores it. Always finish the grace period with SIGKILL.+ self._signal_process_group(process_group_id, signal.SIGKILL)++ @staticmethod+ def _signal_process_group(process_group_id: int, sig: int) -> None:+ try:+ os.killpg(process_group_id, sig)+ except OSError:+ pass++ @staticmethod+ def _reap_timed_out_process(+ process: subprocess.Popen, timeout: float = 1.0+ ) -> None:+ try:+ process.wait(timeout=timeout)+ except subprocess.TimeoutExpired:+ try:+ process.kill()+ except OSError:+ pass+ try:+ process.wait(timeout=timeout)+ except subprocess.TimeoutExpired:+ logger.warning(+ "Timed-out command pid=%s did not exit after SIGKILL",+ process.pid,+ )+ finally:+ for pipe in (process.stdout, process.stderr):+ if pipe is not None:+ try:+ pipe.close()+ except OSError:+ pass+logger = logging.getLogger(__name__)diff --git a/docreader/tests/test_sandbox_executor.py b/docreader/tests/test_sandbox_executor.pynew file mode 100644index 000000000..6347785cd--- /dev/null+++ b/docreader/tests/test_sandbox_executor.py@@ -0,0 +1,292 @@+import os+import shlex+import signal+import subprocess+import sys+import tempfile+import time+import unittest+from pathlib import Path+from unittest.mock import ANY, Mock, call, patch++from docreader.parser.doc_parser import SandboxExecutor+++def _mock_process(pid=4321):+ process = Mock(pid=pid)+ process.stdout = Mock()+ process.stderr = Mock()+ return process++[email protected](os.name == "posix", "process-group cleanup is POSIX-specific")+class TestSandboxExecutorProcessGroup(unittest.TestCase):+ def test_timeout_kills_whole_process_group(self):+ """A timed-out command must terminate its parent and descendants."""+ with tempfile.TemporaryDirectory() as temp_dir:+ child_pid_file = Path(temp_dir) / "child.pid"+ command = [+ "sh",+ "-c",+ f"sleep 30 & echo $! > {shlex.quote(str(child_pid_file))}; wait",+ ]+ executor = SandboxExecutor(default_timeout=1)++ with self.assertRaises(RuntimeError):+ executor.execute_in_sandbox(command)++ child_pid = int(child_pid_file.read_text().strip())+ deadline = time.monotonic() + 1+ while True:+ try:+ os.kill(child_pid, 0)+ except ProcessLookupError:+ break+ if time.monotonic() >= deadline:+ self.fail("timed-out command left a child process running")+ time.sleep(0.01)++ def test_timeout_returns_when_descendant_leaves_process_group(self):+ """setsid descendants can survive killpg; cleanup must still return."""+ with tempfile.TemporaryDirectory() as temp_dir:+ child_pid_file = Path(temp_dir) / "child.pid"+ command = [+ sys.executable,+ "-c",+ (+ "import os, time\n"+ f"pid_file = {str(child_pid_file)!r}\n"+ "if os.fork() == 0:\n"+ " os.setsid()\n"+ " with open(pid_file, 'w', encoding='utf-8') as handle:\n"+ " handle.write(str(os.getpid()))\n"+ " time.sleep(60)\n"+ "time.sleep(30)\n"+ ),+ ]+ executor = SandboxExecutor(default_timeout=1)+ started = time.monotonic()+ with self.assertRaises(RuntimeError):+ executor.execute_in_sandbox(command)+ self.assertLess(+ time.monotonic() - started,+ 4,+ "timeout cleanup hung waiting on an escaped descendant",+ )++ deadline = time.monotonic() + 2+ while not child_pid_file.exists() or not child_pid_file.stat().st_size:+ if time.monotonic() >= deadline:+ self.fail("escaped child did not write its pid file")+ time.sleep(0.01)+ child_pid = int(child_pid_file.read_text().strip())+ try:+ os.kill(child_pid, 0)+ except ProcessLookupError:+ self.fail("expected setsid descendant to survive process-group kill")+ try:+ os.kill(child_pid, signal.SIGKILL)+ except ProcessLookupError:+ pass+++class TestSandboxExecutorMock(unittest.TestCase):+ @unittest.skipUnless(os.name == "posix", "POSIX signals are required")+ def test_posix_timeout_terminates_group_and_reaps_process(self):+ process = _mock_process()+ process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]+ process.wait.side_effect = [+ subprocess.TimeoutExpired(["soffice"], 0.5),+ 0,+ ]++ with (+ patch("docreader.parser.doc_parser.os.name", "posix"),+ patch(+ "docreader.parser.doc_parser.subprocess.Popen", return_value=process+ ) as popen,+ patch("docreader.parser.doc_parser.os.killpg") as killpg,+ patch(+ "docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]+ ),+ patch("docreader.parser.doc_parser.time.sleep") as sleep,+ ):+ with self.assertRaisesRegex(RuntimeError, "timeout after 1"):+ SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])++ popen.assert_called_once_with(+ ["soffice"],+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ env=ANY,+ start_new_session=True,+ )+ killpg.assert_has_calls(+ [+ call(4321, signal.SIGTERM),+ call(4321, signal.SIGKILL),+ ]+ )+ sleep.assert_not_called()+ self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])+ self.assertEqual(+ process.wait.call_args_list,+ [call(timeout=0.5), call(timeout=1.0)],+ )+ process.stdout.close.assert_called_once_with()+ process.stderr.close.assert_called_once_with()++ @unittest.skipUnless(os.name == "posix", "POSIX signals are required")+ def test_posix_timeout_waits_full_grace_period_when_child_exits_early(self):+ process = _mock_process()+ process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]+ process.wait.side_effect = [0, 0]++ with (+ patch("docreader.parser.doc_parser.os.name", "posix"),+ patch(+ "docreader.parser.doc_parser.subprocess.Popen", return_value=process+ ),+ patch("docreader.parser.doc_parser.os.killpg") as killpg,+ patch(+ "docreader.parser.doc_parser.time.monotonic", side_effect=[10, 10.2]+ ),+ patch("docreader.parser.doc_parser.time.sleep") as sleep,+ ):+ with self.assertRaisesRegex(RuntimeError, "timeout after 1"):+ SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])++ self.assertAlmostEqual(sleep.call_args.args[0], 0.3)+ killpg.assert_has_calls(+ [+ call(4321, signal.SIGTERM),+ call(4321, signal.SIGKILL),+ ]+ )+ self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])+ self.assertEqual(+ process.wait.call_args_list,+ [call(timeout=0.5), call(timeout=1.0)],+ )++ @unittest.skipUnless(os.name == "posix", "POSIX signals are required")+ def test_posix_timeout_reaps_when_group_already_exited(self):+ process = _mock_process()+ process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]+ process.wait.side_effect = [+ subprocess.TimeoutExpired(["soffice"], 0.5),+ 0,+ ]++ with (+ patch("docreader.parser.doc_parser.os.name", "posix"),+ patch(+ "docreader.parser.doc_parser.subprocess.Popen", return_value=process+ ),+ patch(+ "docreader.parser.doc_parser.os.killpg",+ side_effect=ProcessLookupError,+ ) as killpg,+ patch(+ "docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]+ ),+ patch("docreader.parser.doc_parser.time.sleep") as sleep,+ ):+ with self.assertRaisesRegex(RuntimeError, "timeout after 1"):+ SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])++ self.assertEqual(killpg.call_count, 2)+ sleep.assert_not_called()+ self.assertEqual(process.wait.call_args_list[-1], call(timeout=1.0))++ @unittest.skipUnless(os.name == "posix", "POSIX signals are required")+ def test_posix_timeout_reaps_when_killpg_raises_permission_error(self):+ process = _mock_process()+ process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]+ process.wait.side_effect = [+ subprocess.TimeoutExpired(["soffice"], 0.5),+ 0,+ ]++ with (+ patch("docreader.parser.doc_parser.os.name", "posix"),+ patch(+ "docreader.parser.doc_parser.subprocess.Popen", return_value=process+ ),+ patch(+ "docreader.parser.doc_parser.os.killpg",+ side_effect=PermissionError,+ ) as killpg,+ patch(+ "docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]+ ),+ patch("docreader.parser.doc_parser.time.sleep") as sleep,+ ):+ with self.assertRaisesRegex(RuntimeError, "timeout after 1"):+ SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])++ self.assertEqual(killpg.call_count, 2)+ sleep.assert_not_called()+ self.assertEqual(process.wait.call_args_list[-1], call(timeout=1.0))+ process.stdout.close.assert_called_once_with()++ @unittest.skipUnless(os.name == "posix", "POSIX signals are required")+ def test_posix_timeout_returns_when_final_wait_times_out(self):+ process = _mock_process()+ process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]+ process.wait.side_effect = [+ subprocess.TimeoutExpired(["soffice"], 0.5),+ subprocess.TimeoutExpired(["soffice"], 1.0),+ subprocess.TimeoutExpired(["soffice"], 1.0),+ ]++ with (+ patch("docreader.parser.doc_parser.os.name", "posix"),+ patch(+ "docreader.parser.doc_parser.subprocess.Popen", return_value=process+ ),+ patch("docreader.parser.doc_parser.os.killpg"),+ patch(+ "docreader.parser.doc_parser.time.monotonic", side_effect=[0, 0.5]+ ),+ patch("docreader.parser.doc_parser.time.sleep"),+ ):+ with self.assertRaisesRegex(RuntimeError, "timeout after 1"):+ SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])++ self.assertEqual(process.kill.call_count, 1)+ process.stdout.close.assert_called_once_with()+ process.stderr.close.assert_called_once_with()++ def test_non_posix_timeout_kills_direct_child_and_reaps_process(self):+ process = _mock_process()+ process.communicate.side_effect = [subprocess.TimeoutExpired(["soffice"], 1)]+ process.wait.return_value = 0++ with (+ patch("docreader.parser.doc_parser.os.name", "nt"),+ patch(+ "docreader.parser.doc_parser.subprocess.Popen", return_value=process+ ) as popen,+ patch("docreader.parser.doc_parser.os.killpg", create=True) as killpg,+ ):+ with self.assertRaisesRegex(RuntimeError, "timeout after 1"):+ SandboxExecutor(default_timeout=1)._execute_with_proxy(["soffice"])++ popen.assert_called_once_with(+ ["soffice"],+ stdout=subprocess.PIPE,+ stderr=subprocess.PIPE,+ env=ANY,+ start_new_session=False,+ )+ process.kill.assert_called_once_with()+ killpg.assert_not_called()+ self.assertEqual(process.communicate.call_args_list, [call(timeout=1)])+ self.assertEqual(process.wait.call_args_list, [call(timeout=1.0)])+ process.stdout.close.assert_called_once_with()+ process.stderr.close.assert_called_once_with()+++if __name__ == "__main__":+ unittest.main()