Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

bench: a failed engine stop is loud and fails the run (#3438): stop_engine returns why the process may still be alive (taskkill result and exit, the 60 s and 30 s timeouts), the run prints an ERROR, writes ENGINE-NOT-STOPPED.txt, exits 6 and reports ok false

Niko1221f8f4e8a7dfdd
diff --git a/strata_cli/bench_standard.py b/strata_cli/bench_standard.py
index 9ee5fb351..5b224414f 100644
--- a/strata_cli/bench_standard.py
+++ b/strata_cli/bench_standard.py
@@ -684,13 +684,14 @@ def start_engine(ref: str, target: Target, log_path: Path, log, timeout_s: float
proc._strata_log = handle # closed by stop_engine
log("starting the engine (the model load is not part of the benchmark time) ...")
if not _start_wait(target, proc, timeout_s, log, log_path):
- stop_engine(proc)
+ stuck = stop_engine(proc)
tail = ""
try:
tail = log_path.read_text(encoding="utf-8", errors="replace")[-600:]
except OSError:
pass
- raise ProfileError("the engine did not become ready" + (": " + tail.strip().splitlines()[-1] if tail.strip() else ""), 6)
+ raise ProfileError("the engine did not become ready" + (": " + tail.strip().splitlines()[-1] if tail.strip() else "") +
+ ("; and it could not be stopped (" + stuck + "): it may still hold the port" if stuck else ""), 6)
return proc
@@ -712,14 +713,25 @@ def engine_log_since(console_log: Path, run_log: Path | None, offset: int) -> Pa
return console_log
-def stop_engine(proc) -> None:
- """Stop only the process this run started."""
+def stop_engine(proc) -> str | None:
+ """Stop only the process this run started. -> None when it is gone, else a plain sentence on why it may still be running
+ (a failed taskkill / signal, or a process that outlives the wait): the caller must not report a normal finish then, because
+ a process that is still alive can hold the server's port and the GPU (#3438)."""
if proc is None:
- return
+ return None
+ problem = None
try:
if proc.poll() is None:
if os.name == "nt":
- subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60)
+ try:
+ done = subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60)
+ if done.returncode != 0 and proc.poll() is None:
+ said = (done.stderr or done.stdout or b"").decode("utf-8", errors="replace").strip()
+ problem = f"taskkill for process {proc.pid} failed (exit {done.returncode}){': ' + said[:200] if said else ''}"
+ except subprocess.TimeoutExpired:
+ problem = f"taskkill for process {proc.pid} did not finish in 60 s"
+ except OSError as error:
+ problem = f"taskkill could not run for process {proc.pid}: {error}"
else:
import signal
try:
@@ -737,10 +749,13 @@ def stop_engine(proc) -> None:
proc.wait(30)
except subprocess.TimeoutExpired:
pass
+ if proc.poll() is None:
+ problem = (problem + "; " if problem else "") + f"process {proc.pid} is still running 30 s after it was told to stop"
finally:
handle = getattr(proc, "_strata_log", None)
if handle is not None:
handle.close()
+ return problem
def slug(text: str, limit: int = 40) -> str:
diff --git a/strata_cli/cli.py b/strata_cli/cli.py
index 97b8554ab..d80f2d5b2 100644
--- a/strata_cli/cli.py
+++ b/strata_cli/cli.py
@@ -1282,6 +1282,7 @@ def _bench_standard(a: argparse.Namespace, machine: bool) -> int:
import tempfile
lease = benchmark_lease(saved["id"], out) if saved else contextlib.nullcontext()
proc, scratch, engine_log, run_log, log_start = None, None, None, None, 0
+ stop_problem = None
try:
with lease, contextlib.redirect_stdout(sys.stderr):
def log(message: str) -> None:
@@ -1301,13 +1302,23 @@ def _bench_standard(a: argparse.Namespace, machine: bool) -> int:
engine_run_log=(run_log, log_start) if proc is not None else None,
started_engine=proc is not None, hash_model=not a.skip_model_hash)
finally:
- bench_standard.stop_engine(proc)
+ stop_problem = bench_standard.stop_engine(proc)
+ if not isinstance(stop_problem, str):
+ stop_problem = None
if scratch is not None:
shutil.rmtree(scratch, ignore_errors=True)
ok = doc["run"]["status"] == "complete"
+ if stop_problem: # #3438: a stop that failed is not a normal finish: the engine may still hold the port and the GPU
+ ok = False
+ print(f"\nERROR: the engine this run started could not be stopped: {stop_problem}. It may still hold the server port "
+ "and the GPU: stop it by hand before the next run. This run is marked FAILED.", file=sys.stderr, flush=True)
+ try:
+ (out_dir / "ENGINE-NOT-STOPPED.txt").write_text(stop_problem + "\n", encoding="utf-8")
+ except OSError:
+ pass
path = out_dir / "results.json"
if machine:
- print(json.dumps({"schema_version": 1, "ok": ok, "path": str(path.resolve()), "status": doc["run"]["status"],
+ print(json.dumps({"schema_version": 1, "ok": ok, "path": str(path.resolve()), "status": "failed" if stop_problem else doc["run"]["status"], "engine_stop_error": stop_problem,
"quality": doc["quality"], "summaries": doc["summaries"]}, ensure_ascii=False, allow_nan=False))
else:
print(bench_standard.format_plain(doc))
diff --git a/tests/terminal/test_bench_standard.py b/tests/terminal/test_bench_standard.py
index bce01fa08..ae2641002 100644
--- a/tests/terminal/test_bench_standard.py
+++ b/tests/terminal/test_bench_standard.py
@@ -936,6 +936,97 @@ class EngineProcessTests(unittest.TestCase):
else:
killpg.assert_called()
+ # #3438: the stop used to ignore taskkill's result and the wait timeout, so a stop that failed still ended as a normal finish
+ def _stuck_proc(self):
+ class Stuck:
+ pid = 777
+
+ def poll(self):
+ return None # never exits
+
+ def wait(self, timeout=None):
+ raise bs.subprocess.TimeoutExpired("x", timeout)
+
+ def terminate(self):
+ pass
+
+ def kill(self):
+ pass
+ return Stuck()
+
+ def test_stop_engine_says_so_when_the_process_survives(self):
+ proc = self._stuck_proc()
+ done = bs.subprocess.CompletedProcess([], 1, b"", b"ERROR: Access is denied.")
+ with mock.patch("subprocess.run", return_value=done), mock.patch("os.killpg", create=True):
+ problem = bs.stop_engine(proc)
+ self.assertIsNotNone(problem)
+ self.assertIn("777", problem)
+ self.assertIn("still running", problem)
+ if os.name == "nt":
+ self.assertIn("taskkill", problem)
+ self.assertIn("Access is denied", problem)
+
+ @unittest.skipUnless(os.name == "nt", "taskkill is the Windows stop")
+ def test_stop_engine_reports_a_taskkill_that_times_out(self):
+ proc = self._stuck_proc()
+ with mock.patch("subprocess.run", side_effect=bs.subprocess.TimeoutExpired("taskkill", 60)):
+ problem = bs.stop_engine(proc)
+ self.assertIn("did not finish in 60 s", problem)
+
+ def test_stop_engine_is_quiet_when_the_process_ends(self):
+ class Ends:
+ pid = 5
+ alive = True
+
+ def poll(self):
+ return None if self.alive else 0
+
+ def wait(self, timeout=None):
+ self.alive = False
+ return 0
+
+ def terminate(self):
+ self.alive = False
+ proc = Ends()
+ done = bs.subprocess.CompletedProcess([], 0, b"", b"")
+
+ def kill(*args, **kwargs):
+ proc.alive = False
+ return done
+ with mock.patch("subprocess.run", side_effect=kill), mock.patch("os.killpg", create=True, side_effect=lambda *a: setattr(proc, "alive", False)):
+ self.assertIsNone(bs.stop_engine(proc))
+ self.assertIsNone(bs.stop_engine(None))
+ # a process that exited by itself needs no kill at all
+ gone = mock.Mock()
+ gone.poll.return_value = 0
+ self.assertIsNone(bs.stop_engine(gone))
+
+ def test_a_run_whose_engine_cannot_be_stopped_is_marked_failed_and_loud(self):
+ # cli: the run finished, the stop failed -> exit code 6, an ERROR line on stderr, a note in the folder, ok false in the json
+ plan = {"effective": {"host": "127.0.0.1", "port": 1}, "engine_argv": [], "config": {}}
+ fake_doc = {"run": {"status": "complete"}, "quality": {}, "summaries": []}
+ out_dir = Path(tempfile.mkdtemp())
+ self.addCleanup(shutil.rmtree, out_dir, True)
+ (out_dir / "results.json").write_text("{}", encoding="utf-8")
+ (out_dir / "README.md").write_text("x", encoding="utf-8")
+ with mock.patch("strata_cli.cli._source", return_value=(Path("p.json"), None, None, {})), \
+ mock.patch("strata_cli.cli._launch_plan", return_value=(plan, [])), \
+ mock.patch.object(bs, "start_engine", return_value=object()), \
+ mock.patch.object(bs, "run", return_value=(fake_doc, out_dir)), \
+ mock.patch.object(bs, "stop_engine", return_value="process 9 is still running 30 s after it was told to stop"), \
+ mock.patch("strata_cli.client.server_url", side_effect=lambda u: u):
+ err = io.StringIO()
+ out = io.StringIO()
+ with contextlib.redirect_stderr(err), contextlib.redirect_stdout(out):
+ try:
+ code = main(["bench", "run", "--suite", "standard-v1", "--profile", "p.json", "--out", str(out_dir), "--output", "json"])
+ except SystemExit as stop:
+ code = stop.code
+ self.assertEqual(code, 6)
+ self.assertIn("could not be stopped", err.getvalue())
+ self.assertIn("FAILED", err.getvalue())
+ self.assertTrue((out_dir / "ENGINE-NOT-STOPPED.txt").is_file())
+
def test_an_engine_that_exits_before_it_is_ready_is_an_error(self):
class Dead:
pid = 1