Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
bench: a failed engine stop is loud and fails the run (#3438): stop_engine returns why the process may still be alive (taskkill result and exit, the 60 s and 30 s timeouts), the run prints an ERROR, writes ENGINE-NOT-STOPPED.txt, exits 6 and reports ok false
Niko1221f8f4e8a7dfdd
diff --git a/strata_cli/bench_standard.py b/strata_cli/bench_standard.pyindex 9ee5fb351..5b224414f 100644--- a/strata_cli/bench_standard.py+++ b/strata_cli/bench_standard.py@@ -684,13 +684,14 @@ def start_engine(ref: str, target: Target, log_path: Path, log, timeout_s: floatproc._strata_log = handle # closed by stop_enginelog("starting the engine (the model load is not part of the benchmark time) ...")if not _start_wait(target, proc, timeout_s, log, log_path):- stop_engine(proc)+ stuck = stop_engine(proc)tail = ""try:tail = log_path.read_text(encoding="utf-8", errors="replace")[-600:]except OSError:pass- raise ProfileError("the engine did not become ready" + (": " + tail.strip().splitlines()[-1] if tail.strip() else ""), 6)+ raise ProfileError("the engine did not become ready" + (": " + tail.strip().splitlines()[-1] if tail.strip() else "") ++ ("; and it could not be stopped (" + stuck + "): it may still hold the port" if stuck else ""), 6)return proc@@ -712,14 +713,25 @@ def engine_log_since(console_log: Path, run_log: Path | None, offset: int) -> Pareturn console_log-def stop_engine(proc) -> None:- """Stop only the process this run started."""+def stop_engine(proc) -> str | None:+ """Stop only the process this run started. -> None when it is gone, else a plain sentence on why it may still be running+ (a failed taskkill / signal, or a process that outlives the wait): the caller must not report a normal finish then, because+ a process that is still alive can hold the server's port and the GPU (#3438)."""if proc is None:- return+ return None+ problem = Nonetry:if proc.poll() is None:if os.name == "nt":- subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60)+ try:+ done = subprocess.run(["taskkill", "/PID", str(proc.pid), "/T", "/F"], capture_output=True, timeout=60)+ if done.returncode != 0 and proc.poll() is None:+ said = (done.stderr or done.stdout or b"").decode("utf-8", errors="replace").strip()+ problem = f"taskkill for process {proc.pid} failed (exit {done.returncode}){': ' + said[:200] if said else ''}"+ except subprocess.TimeoutExpired:+ problem = f"taskkill for process {proc.pid} did not finish in 60 s"+ except OSError as error:+ problem = f"taskkill could not run for process {proc.pid}: {error}"else:import signaltry:@@ -737,10 +749,13 @@ def stop_engine(proc) -> None:proc.wait(30)except subprocess.TimeoutExpired:pass+ if proc.poll() is None:+ problem = (problem + "; " if problem else "") + f"process {proc.pid} is still running 30 s after it was told to stop"finally:handle = getattr(proc, "_strata_log", None)if handle is not None:handle.close()+ return problemdef slug(text: str, limit: int = 40) -> str:diff --git a/strata_cli/cli.py b/strata_cli/cli.pyindex 97b8554ab..d80f2d5b2 100644--- a/strata_cli/cli.py+++ b/strata_cli/cli.py@@ -1282,6 +1282,7 @@ def _bench_standard(a: argparse.Namespace, machine: bool) -> int:import tempfilelease = benchmark_lease(saved["id"], out) if saved else contextlib.nullcontext()proc, scratch, engine_log, run_log, log_start = None, None, None, None, 0+ stop_problem = Nonetry:with lease, contextlib.redirect_stdout(sys.stderr):def log(message: str) -> None:@@ -1301,13 +1302,23 @@ def _bench_standard(a: argparse.Namespace, machine: bool) -> int:engine_run_log=(run_log, log_start) if proc is not None else None,started_engine=proc is not None, hash_model=not a.skip_model_hash)finally:- bench_standard.stop_engine(proc)+ stop_problem = bench_standard.stop_engine(proc)+ if not isinstance(stop_problem, str):+ stop_problem = Noneif scratch is not None:shutil.rmtree(scratch, ignore_errors=True)ok = doc["run"]["status"] == "complete"+ if stop_problem: # #3438: a stop that failed is not a normal finish: the engine may still hold the port and the GPU+ ok = False+ print(f"\nERROR: the engine this run started could not be stopped: {stop_problem}. It may still hold the server port "+ "and the GPU: stop it by hand before the next run. This run is marked FAILED.", file=sys.stderr, flush=True)+ try:+ (out_dir / "ENGINE-NOT-STOPPED.txt").write_text(stop_problem + "\n", encoding="utf-8")+ except OSError:+ passpath = out_dir / "results.json"if machine:- print(json.dumps({"schema_version": 1, "ok": ok, "path": str(path.resolve()), "status": doc["run"]["status"],+ print(json.dumps({"schema_version": 1, "ok": ok, "path": str(path.resolve()), "status": "failed" if stop_problem else doc["run"]["status"], "engine_stop_error": stop_problem,"quality": doc["quality"], "summaries": doc["summaries"]}, ensure_ascii=False, allow_nan=False))else:print(bench_standard.format_plain(doc))diff --git a/tests/terminal/test_bench_standard.py b/tests/terminal/test_bench_standard.pyindex bce01fa08..ae2641002 100644--- a/tests/terminal/test_bench_standard.py+++ b/tests/terminal/test_bench_standard.py@@ -936,6 +936,97 @@ class EngineProcessTests(unittest.TestCase):else:killpg.assert_called()+ # #3438: the stop used to ignore taskkill's result and the wait timeout, so a stop that failed still ended as a normal finish+ def _stuck_proc(self):+ class Stuck:+ pid = 777++ def poll(self):+ return None # never exits++ def wait(self, timeout=None):+ raise bs.subprocess.TimeoutExpired("x", timeout)++ def terminate(self):+ pass++ def kill(self):+ pass+ return Stuck()++ def test_stop_engine_says_so_when_the_process_survives(self):+ proc = self._stuck_proc()+ done = bs.subprocess.CompletedProcess([], 1, b"", b"ERROR: Access is denied.")+ with mock.patch("subprocess.run", return_value=done), mock.patch("os.killpg", create=True):+ problem = bs.stop_engine(proc)+ self.assertIsNotNone(problem)+ self.assertIn("777", problem)+ self.assertIn("still running", problem)+ if os.name == "nt":+ self.assertIn("taskkill", problem)+ self.assertIn("Access is denied", problem)++ @unittest.skipUnless(os.name == "nt", "taskkill is the Windows stop")+ def test_stop_engine_reports_a_taskkill_that_times_out(self):+ proc = self._stuck_proc()+ with mock.patch("subprocess.run", side_effect=bs.subprocess.TimeoutExpired("taskkill", 60)):+ problem = bs.stop_engine(proc)+ self.assertIn("did not finish in 60 s", problem)++ def test_stop_engine_is_quiet_when_the_process_ends(self):+ class Ends:+ pid = 5+ alive = True++ def poll(self):+ return None if self.alive else 0++ def wait(self, timeout=None):+ self.alive = False+ return 0++ def terminate(self):+ self.alive = False+ proc = Ends()+ done = bs.subprocess.CompletedProcess([], 0, b"", b"")++ def kill(*args, **kwargs):+ proc.alive = False+ return done+ with mock.patch("subprocess.run", side_effect=kill), mock.patch("os.killpg", create=True, side_effect=lambda *a: setattr(proc, "alive", False)):+ self.assertIsNone(bs.stop_engine(proc))+ self.assertIsNone(bs.stop_engine(None))+ # a process that exited by itself needs no kill at all+ gone = mock.Mock()+ gone.poll.return_value = 0+ self.assertIsNone(bs.stop_engine(gone))++ def test_a_run_whose_engine_cannot_be_stopped_is_marked_failed_and_loud(self):+ # cli: the run finished, the stop failed -> exit code 6, an ERROR line on stderr, a note in the folder, ok false in the json+ plan = {"effective": {"host": "127.0.0.1", "port": 1}, "engine_argv": [], "config": {}}+ fake_doc = {"run": {"status": "complete"}, "quality": {}, "summaries": []}+ out_dir = Path(tempfile.mkdtemp())+ self.addCleanup(shutil.rmtree, out_dir, True)+ (out_dir / "results.json").write_text("{}", encoding="utf-8")+ (out_dir / "README.md").write_text("x", encoding="utf-8")+ with mock.patch("strata_cli.cli._source", return_value=(Path("p.json"), None, None, {})), \+ mock.patch("strata_cli.cli._launch_plan", return_value=(plan, [])), \+ mock.patch.object(bs, "start_engine", return_value=object()), \+ mock.patch.object(bs, "run", return_value=(fake_doc, out_dir)), \+ mock.patch.object(bs, "stop_engine", return_value="process 9 is still running 30 s after it was told to stop"), \+ mock.patch("strata_cli.client.server_url", side_effect=lambda u: u):+ err = io.StringIO()+ out = io.StringIO()+ with contextlib.redirect_stderr(err), contextlib.redirect_stdout(out):+ try:+ code = main(["bench", "run", "--suite", "standard-v1", "--profile", "p.json", "--out", str(out_dir), "--output", "json"])+ except SystemExit as stop:+ code = stop.code+ self.assertEqual(code, 6)+ self.assertIn("could not be stopped", err.getvalue())+ self.assertIn("FAILED", err.getvalue())+ self.assertTrue((out_dir / "ENGINE-NOT-STOPPED.txt").is_file())+def test_an_engine_that_exits_before_it_is_ready_is_an_error(self):class Dead:pid = 1