Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(tools): keep datapoint results when engine teardown outlasts the grace period

JustVugg21e80f3f0fcb
diff --git a/c/openai_server.py b/c/openai_server.py
index b554e973..619011b9 100644
--- a/c/openai_server.py
+++ b/c/openai_server.py
@@ -2381,8 +2381,17 @@ class Engine:
try:
self.process.wait(timeout=5)
except subprocess.TimeoutExpired:
+ # A large resident cache (e.g. 111 GB at --memory-gb 126) can
+ # take longer than the grace period to unmap and free on
+ # SIGTERM. SIGKILL cannot be caught, so the process is already
+ # on its way out; a second timeout only means the reap has not
+ # landed yet. Teardown is best-effort: never raise from here, or
+ # a completed measurement is lost to a shutdown that succeeded.
self.process.kill()
- self.process.wait(timeout=5)
+ try:
+ self.process.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ pass
if self.dispatcher is not threading.current_thread():
self.dispatcher.join(timeout=5)
diff --git a/c/tests/test_datapoint.py b/c/tests/test_datapoint.py
index f5f069b4..bb3e350d 100644
--- a/c/tests/test_datapoint.py
+++ b/c/tests/test_datapoint.py
@@ -7,6 +7,7 @@ mocking sys.platform and the two win32 probes.
"""
import ctypes
import os
+import subprocess
import sys
import tempfile
import threading
@@ -154,6 +155,34 @@ class PersistentDatapointTest(unittest.TestCase):
self.assertEqual(campaign["rotating_prompt_count"], 4)
self.assertEqual(campaign["rotating_prompt_source"], "built-in")
+ def test_teardown_failure_does_not_discard_collected_results(self):
+ # At --memory-gb 126 the engine can outlast the close() grace period
+ # while it frees a large resident cache; the raised TimeoutExpired used
+ # to swallow the return and lose a complete run (#1154). A teardown that
+ # fails must still yield the measurements already gathered.
+ base = self._runtime()
+
+ class RaisingCloseEngine(base.Engine):
+ def close(self):
+ self.closed = True
+ raise subprocess.TimeoutExpired(cmd="engine", timeout=5)
+
+ runtime = self._runtime(engine_type=RaisingCloseEngine)
+ with tempfile.TemporaryDirectory() as tmp:
+ executable = Path(tmp) / "deepseek_v4"
+ executable.touch()
+ with mock.patch.dict(os.environ, {}, clear=True):
+ campaign = datapoint.run_persistent_engine(
+ str(executable), "/model", "hello", 8, 0, 1, 16,
+ memory_gb=126, runtime=runtime, physical_cores=8)
+
+ self.assertEqual(len(self.instances), 1)
+ self.assertTrue(self.instances[0].closed)
+ self.assertEqual(len(campaign["cold"]), 1)
+ self.assertEqual(len(campaign["warm"]), 1)
+ self.assertEqual([row["tok_s"] for row in campaign["rotating"]],
+ [3.0, 4.0, 5.0, 6.0])
+
def test_engine_is_closed_when_a_request_fails(self):
instances = self.instances
diff --git a/c/tools/datapoint.py b/c/tools/datapoint.py
index ec155ca9..664c8f34 100644
--- a/c/tools/datapoint.py
+++ b/c/tools/datapoint.py
@@ -353,7 +353,16 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,
tiers = dict(persistent.tiers) if getattr(persistent, "tiers", None) else None
hwinfo = dict(persistent.hwinfo) if getattr(persistent, "hwinfo", None) else None
finally:
- persistent.close()
+ # All measurements above are already collected; a teardown that fails
+ # must not discard them. At --memory-gb 126 the engine can outlast the
+ # close() grace period while it frees a 111 GB resident cache, and a
+ # raised exception here would swallow the return and lose a complete run
+ # (observed on the EPYC 7282 datapoint, #1154).
+ try:
+ persistent.close()
+ except Exception as exc: # noqa: BLE001 - teardown is best-effort
+ print(f"[datapoint] warning: engine teardown did not complete "
+ f"cleanly, results are still valid: {exc}", file=sys.stderr)
return {
"mode": "persistent",