Skip to content

DeepTutor: Lifelong Personalized Tutoring. https://deeptutor.info/.

fix: reap failed engine workers before notifying callers

Bingxi Zhao (Frank)ce333434c31a
diff --git a/deeptutor/runtime/ts_engine/client.py b/deeptutor/runtime/ts_engine/client.py
index 397e74808..c802259c7 100644
--- a/deeptutor/runtime/ts_engine/client.py
+++ b/deeptutor/runtime/ts_engine/client.py
@@ -97,12 +97,14 @@ class EngineClient:
if process.returncode is None:
with suppress(ProcessLookupError):
process.terminate()
+ # A caller may start another turn as soon as it receives failure.
+ # Reap first so _ensure_started cannot reuse the exiting worker.
+ await process.wait()
for queue in tuple(self._runs.values()):
with suppress(asyncio.QueueFull):
queue.put_nowait(
EngineUnavailableError("TypeScript scheduler stopped during turn")
)
- await process.wait()
async def _send(self, frame: dict[str, Any]) -> None:
payload = (json.dumps({"v": 1, **frame}, ensure_ascii=False) + "\n").encode()
diff --git a/tests/runtime/test_ts_engine_lifecycle.py b/tests/runtime/test_ts_engine_lifecycle.py
new file mode 100644
index 000000000..69d6c23c3
--- /dev/null
+++ b/tests/runtime/test_ts_engine_lifecycle.py
@@ -0,0 +1,44 @@
+"""Worker loss must be observable only after that worker can no longer be reused."""
+
+import asyncio
+from types import SimpleNamespace
+from unittest.mock import AsyncMock
+
+import pytest
+
+from deeptutor.runtime.ts_engine.client import EngineClient, EngineUnavailableError
+
+
[email protected]("last_frame", [b"", b"invalid protocol\n"])
+async def test_worker_failure_waits_for_process_exit_before_notifying_turns(last_frame):
+ reaping = asyncio.Event()
+ exited = asyncio.Event()
+ process = SimpleNamespace(
+ stdout=SimpleNamespace(readline=AsyncMock(return_value=last_frame)),
+ returncode=None,
+ terminate=lambda: None,
+ )
+
+ async def wait():
+ reaping.set()
+ await exited.wait()
+ process.returncode = -15
+ return process.returncode
+
+ process.wait = wait
+ client = EngineClient()
+ client._process = process
+ queue = asyncio.Queue()
+ client._runs["turn"] = queue
+ reader = asyncio.create_task(client._read(process))
+ try:
+ await asyncio.wait_for(reaping.wait(), 1)
+ # A notified caller may immediately start another turn. While returncode
+ # is None, _ensure_started would still consider this worker reusable.
+ assert queue.empty()
+ finally:
+ exited.set()
+ await reader
+ assert process.returncode is not None
+ assert isinstance(queue.get_nowait(), EngineUnavailableError)