Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(tools): keep datapoint results when engine teardown outlasts the grace period
JustVugg21e80f3f0fcb
diff --git a/c/openai_server.py b/c/openai_server.pyindex b554e973..619011b9 100644--- a/c/openai_server.py+++ b/c/openai_server.py@@ -2381,8 +2381,17 @@ class Engine:try:self.process.wait(timeout=5)except subprocess.TimeoutExpired:+ # A large resident cache (e.g. 111 GB at --memory-gb 126) can+ # take longer than the grace period to unmap and free on+ # SIGTERM. SIGKILL cannot be caught, so the process is already+ # on its way out; a second timeout only means the reap has not+ # landed yet. Teardown is best-effort: never raise from here, or+ # a completed measurement is lost to a shutdown that succeeded.self.process.kill()- self.process.wait(timeout=5)+ try:+ self.process.wait(timeout=5)+ except subprocess.TimeoutExpired:+ passif self.dispatcher is not threading.current_thread():self.dispatcher.join(timeout=5)diff --git a/c/tests/test_datapoint.py b/c/tests/test_datapoint.pyindex f5f069b4..bb3e350d 100644--- a/c/tests/test_datapoint.py+++ b/c/tests/test_datapoint.py@@ -7,6 +7,7 @@ mocking sys.platform and the two win32 probes."""import ctypesimport os+import subprocessimport sysimport tempfileimport threading@@ -154,6 +155,34 @@ class PersistentDatapointTest(unittest.TestCase):self.assertEqual(campaign["rotating_prompt_count"], 4)self.assertEqual(campaign["rotating_prompt_source"], "built-in")+ def test_teardown_failure_does_not_discard_collected_results(self):+ # At --memory-gb 126 the engine can outlast the close() grace period+ # while it frees a large resident cache; the raised TimeoutExpired used+ # to swallow the return and lose a complete run (#1154). A teardown that+ # fails must still yield the measurements already gathered.+ base = self._runtime()++ class RaisingCloseEngine(base.Engine):+ def close(self):+ self.closed = True+ raise subprocess.TimeoutExpired(cmd="engine", timeout=5)++ runtime = self._runtime(engine_type=RaisingCloseEngine)+ with tempfile.TemporaryDirectory() as tmp:+ executable = Path(tmp) / "deepseek_v4"+ executable.touch()+ with mock.patch.dict(os.environ, {}, clear=True):+ campaign = datapoint.run_persistent_engine(+ str(executable), "/model", "hello", 8, 0, 1, 16,+ memory_gb=126, runtime=runtime, physical_cores=8)++ self.assertEqual(len(self.instances), 1)+ self.assertTrue(self.instances[0].closed)+ self.assertEqual(len(campaign["cold"]), 1)+ self.assertEqual(len(campaign["warm"]), 1)+ self.assertEqual([row["tok_s"] for row in campaign["rotating"]],+ [3.0, 4.0, 5.0, 6.0])+def test_engine_is_closed_when_a_request_fails(self):instances = self.instancesdiff --git a/c/tools/datapoint.py b/c/tools/datapoint.pyindex ec155ca9..664c8f34 100644--- a/c/tools/datapoint.py+++ b/c/tools/datapoint.py@@ -353,7 +353,16 @@ def run_persistent_engine(engine, snap, prompt, max_new, warmup_runs, warm_runs,tiers = dict(persistent.tiers) if getattr(persistent, "tiers", None) else Nonehwinfo = dict(persistent.hwinfo) if getattr(persistent, "hwinfo", None) else Nonefinally:- persistent.close()+ # All measurements above are already collected; a teardown that fails+ # must not discard them. At --memory-gb 126 the engine can outlast the+ # close() grace period while it frees a 111 GB resident cache, and a+ # raised exception here would swallow the return and lose a complete run+ # (observed on the EPYC 7282 datapoint, #1154).+ try:+ persistent.close()+ except Exception as exc: # noqa: BLE001 - teardown is best-effort+ print(f"[datapoint] warning: engine teardown did not complete "+ f"cleanly, results are still valid: {exc}", file=sys.stderr)return {"mode": "persistent",