Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.
C++
Fix #3683: the test start no longer waits out its 10 minutes when a Windows venv launcher sits above the server
Niko122172d0bf667e50
diff --git a/strata_control/runtime.py b/strata_control/runtime.pyindex 440504a65..2497957d9 100644--- a/strata_control/runtime.py+++ b/strata_control/runtime.py@@ -468,6 +468,30 @@ def _engine_preflight(plan: dict) -> None:raise ProfileError(str(exc), exc.exit_code) from None+def _ready_pid_is_child(ready_pid, child_pid: int) -> bool:+ """Is the process named in the ready file the one this host started?++ Normally the pids are equal. A Windows venv (the python.exe of a source checkout's .venv) starts its real interpreter as a+ CHILD of the launcher it was given, so the server that writes the ready file is one process below the pid ``Popen`` returned+ (#3683: the staged engine was ready, this check never matched, and the start waited out its whole deadline). Only on Windows,+ and only for a process that sits within three parents of the process this host started, so another process cannot pass."""+ if type(ready_pid) is not int or ready_pid <= 0:+ return False+ if ready_pid == child_pid:+ return True+ if not WINDOWS:+ return False+ current = ready_pid+ for _ in range(3):+ parent = procs.parent_of(current)+ if parent is None or parent <= 0:+ return False+ if parent == child_pid:+ return True+ current = parent+ return False++def _check_start_candidate(name: str, expected_profile_revision: str | None,expected_source_sha256: str | None) -> tuple[dict, str]:plan, digest = _start_candidate(name)@@ -481,9 +505,13 @@ def _check_start_candidate(name: str, expected_profile_revision: str | None,def start(name: str, *, wait: float = 300, shutdown_timeout: float = 30,expected_profile_revision: str | None = None,expected_source_sha256: str | None = None,- die_with_pid: int | None = None) -> dict:+ die_with_pid: int | None = None, on_status=None) -> dict:"""Start the saved profile under a new owner host.+ ``on_status`` (optional) is called with each status answer while the start waits (about every 0.2 s; the caller throttles), so a+ caller can show progress. The result of a wait that ran out names the last phase (``loading``, ``verifying_generation``,+ ``ready_file_not_matched``).+The owner host is a background service: it is not tied to the terminal or the process that ran this command andkeeps serving after they end. It stops on `runtime stop`, when its server ends, or - only if `die_with_pid`names a process - when that process is gone (the desktop app uses this for the engines it starts)."""@@ -542,14 +570,20 @@ def start(name: str, *, wait: float = 300, shutdown_timeout: float = 30,if wait == 0:return {"ok": True, "profile": name, "state": "starting", "plan": plan}deadline = time.monotonic() + wait+ result: dict = {}while time.monotonic() < deadline:result = _rpc(name, "status")+ if on_status is not None:+ try:+ on_status(result)+ except Exception: # noqa: BLE001 - progress must never break a start+ on_status = Noneif result.get("state") in ("ready", "failed"):return {**result, "ok": result["state"] == "ready","exit_code": 6 if result["state"] == "failed" else 0}time.sleep(0.2)return {"ok": False, "profile": name, "state": "starting", "error": "startup_wait_expired",- "exit_code": 8}+ "phase": result.get("phase"), "exit_code": 8}def plan_stop(name: str) -> dict:@@ -901,7 +935,7 @@ class Host:if self.ready_path is not None:try:ready = json.loads(self.ready_path.read_text(encoding="utf-8"))- if ready.get("pid") == child.pid and isinstance(ready.get("instance_id"), str):+ if _ready_pid_is_child(ready.get("pid"), child.pid) and isinstance(ready.get("instance_id"), str):key = (child.pid, ready["instance_id"])if self.probe_key != key:self.probe_key, self.probe_result = key, None@@ -920,9 +954,13 @@ class Host:return {**base, "state": "ready", "instance_id": ready["instance_id"],"model_state": ready.get("model_state"),**self.probe_result}+ if isinstance(ready, dict):+ # the server published its ready file but it is not the process this host started: say so, so a wait that+ # ends can name it (before #3683 this looked exactly like a model that was still loading)+ return {**base, "state": "starting", "phase": "ready_file_not_matched"}except (OSError, ValueError):pass- return {**base, "state": "starting"}+ return {**base, "state": "starting", "phase": "loading"}def _wait_ready(self, timeout: float) -> dict:deadline = time.monotonic() + timeout@@ -1139,7 +1177,7 @@ class Host:return {"ok": False, "state": current["state"], "error": "runtime_not_ready", "exit_code": 6}try:ready = json.loads(self.ready_path.read_text(encoding="utf-8"))- if ready.get("pid") != self.child.pid or ready.get("instance_id") != current["instance_id"]:+ if not _ready_pid_is_child(ready.get("pid"), self.child.pid) or ready.get("instance_id") != current["instance_id"]:raise readiness.ReadinessError("probe_instance_mismatch")host = self.args[self.args.index("--host") + 1]result = readiness.verify(ready, self.source_bytes, expected_host=host,diff --git a/strata_control/updater.py b/strata_control/updater.pyindex 0f504ab73..7f4d719e9 100644--- a/strata_control/updater.py+++ b/strata_control/updater.py@@ -76,12 +76,15 @@ class TestStartFailed(UpdateError):BUSY = re.compile(r"timed?[ -]?out|out of memory|\boom\b|not enough (gpu |video |free )?memory|vram|memory allocation|"r"cudaerrormemory|device (is )?(busy|in use)|resource busy|no free (gpu|memory)", re.I)- def __init__(self, label: str, reason: str):- self.busy = bool(self.BUSY.search(reason))+ def __init__(self, label: str, reason: str, *, busy: bool | None = None, hint: str | None = None):+ # busy: True/False says outright whether the card or memory was busy (the wait-expired reasons carry the engine's log tail, which+ # mentions VRAM on every healthy start and must not be read as a busy card, #3683); None reads it from the words of the reason+ self.busy = bool(self.BUSY.search(reason)) if busy is None else busy+ if hint is None and self.busy:+ hint = ("The graphics card or memory looks busy: close programs that use the card (games, video, other models) "+ "and run `strata update` again.")super().__init__(f"The new engine {label} did not pass its test start ({reason}). It was not installed; "- "your current engine is unchanged."- + (" The graphics card or memory looks busy: close programs that use the card (games, video, other models) "- "and run `strata update` again." if self.busy else ""), 6)+ "your current engine is unchanged." + (" " + hint if hint else ""), 6)self.label = labelself.reason = reason@@ -702,6 +705,48 @@ def _warn_if_deep(ctx: Context, folder: Path, longest_inside: int) -> None:ctx.say("warn", words, component="engine")+def _test_progress(ctx: Context, component: ec.EngineComponent):+ """What the person sees while the staged engine is test-started: a line when the phase changes and one about every 30 seconds,+ so a long load or a stuck probe is never a silent wait (#3683: 10 minutes of nothing, then a timeout)."""+ began = time.monotonic()+ seen = {"phase": None, "at": 0.0}+ words = {"loading": "is loading the model",+ "verifying_generation": "has loaded the model; waiting for it to answer one token",+ "ready_file_not_matched": "says it is ready; checking that it is the engine Strata started"}++ def progress(status: dict) -> None:+ phase = status.get("phase") if status.get("state") == "starting" else None+ if phase not in words:+ return+ now = time.monotonic()+ if phase != seen["phase"] or now - seen["at"] >= 30:+ seen["phase"], seen["at"] = phase, now+ ctx.say("test", f"Engine {component.version} {words[phase]} ({int(now - began)} s)", component="engine")++ return progress+++def _test_start_failure(component: ec.EngineComponent, started: dict, ctx: Context, tail: str) -> TestStartFailed:+ """The plain reason a test start did not pass, in the words of what the engine was doing when it ran out of time."""+ error = started.get("error")+ minutes = max(1, int(min(ctx.test_timeout, 600) // 60))+ log = updates_dir(ctx.components) / "logs" / f"test-start-{component.version}.log"+ if error == "startup_wait_expired":+ phase = started.get("phase")+ kept = f" The engine's log from this test is in {log}."+ if phase == "verifying_generation":+ return TestStartFailed(component.version, f"it loaded the model but did not answer one token within {minutes} minutes",+ busy=False, hint="Nothing was changed. Run `strata update` again; if it happens again, report it with the log." + kept)+ if phase == "ready_file_not_matched":+ return TestStartFailed(component.version, f"it reported that it was ready, but Strata could not match it to the engine process it "+ f"started", busy=False,+ hint="This looks like a problem in Strata, not in your engine or card. Nothing was changed; report it with the log." + kept)+ return TestStartFailed(component.version, f"it was still loading the model after {minutes} minutes", busy=True,+ hint="Nothing was changed. A slow disk or a card that other programs are using can do this: close them and run "+ "`strata update` again." + kept)+ return TestStartFailed(component.version, f"it did not answer a one-token probe ({error or started.get('state')}){tail}")++def default_test_start(stage_root: Path, component: ec.EngineComponent, ctx: Context) -> dict:"""Start the staged engine through the real start path (owner runtime, `component:strata-engine`, a saved model), wait forthe one-token generation probe, and stop it. Everything runs in an isolated runtime, profile and component folder, so a@@ -732,15 +777,16 @@ def default_test_start(stage_root: Path, component: ec.EngineComponent, ctx: Conprofiles.create(name, str(config_path))started = Nonetry:- started = runtime.start(name, wait=min(ctx.test_timeout, 600), shutdown_timeout=20)+ started = runtime.start(name, wait=min(ctx.test_timeout, 600), shutdown_timeout=20,+ on_status=_test_progress(ctx, component))if started.get("ok") and started.get("state") == "ready" and started.get("generation_ready") is not True:# a server that loads its model on the first request is "ready" without having generated anything:# force the load and the one-token probe, because that is what the new engine has to be able to do+ ctx.say("test", f"Engine {component.version} is up; asking it for one token", component="engine")started = {**started, **runtime.verify(name, timeout=min(ctx.test_timeout, 600), load=True)}if not started.get("ok") or started.get("state") != "ready" or started.get("generation_ready") is not True:- raise TestStartFailed(component.version,- f"it did not answer a one-token probe ({started.get('error') or started.get('state')})"- + _log_tail(sandbox, component))+ _keep_log(sandbox, component, ctx)+ raise _test_start_failure(component, started, ctx, _log_tail(sandbox, component))_keep_log(sandbox, component, ctx)return {"mode": "full", "model": cfg.get("model_name") or source.stem, "state": started["state"],"generation_ready": True, "detail": "started, answered a one-token probe, stopped"}diff --git a/tests/terminal/test_runtime.py b/tests/terminal/test_runtime.pyindex c54b9c6a8..7b795b9bc 100644--- a/tests/terminal/test_runtime.py+++ b/tests/terminal/test_runtime.py@@ -1195,6 +1195,53 @@ class RuntimeTests(unittest.TestCase):with self.assertRaisesRegex(profiles.ProfileError, "Everyone"):runtime.runtime_dir()+ def test_ready_pid_may_be_a_child_of_the_launcher_on_windows_only(self):+ # #3683: a Windows venv's python.exe starts the real interpreter as its child, so the server that writes the ready file+ # is one process below the pid the host started. It must still match; a stranger or a distant process must not.+ parents = {200: 100, 300: 200, 400: 300, 500: 400}+ with mock.patch.object(runtime.procs, "parent_of", side_effect=lambda pid: parents.get(pid)):+ with mock.patch.object(runtime, "WINDOWS", True):+ self.assertTrue(runtime._ready_pid_is_child(100, 100))+ self.assertTrue(runtime._ready_pid_is_child(200, 100))+ self.assertTrue(runtime._ready_pid_is_child(300, 100))+ self.assertTrue(runtime._ready_pid_is_child(400, 100))+ self.assertFalse(runtime._ready_pid_is_child(500, 100)) # too far below+ self.assertFalse(runtime._ready_pid_is_child(999, 100)) # unrelated+ self.assertFalse(runtime._ready_pid_is_child(None, 100))+ self.assertFalse(runtime._ready_pid_is_child("200", 100))+ self.assertFalse(runtime._ready_pid_is_child(0, 100))+ with mock.patch.object(runtime, "WINDOWS", False):+ self.assertTrue(runtime._ready_pid_is_child(100, 100))+ self.assertFalse(runtime._ready_pid_is_child(200, 100)) # POSIX has no launcher in between++ @unittest.skipUnless(WINDOWS, "the venv launcher is a Windows thing")+ def test_server_started_through_a_launcher_process_becomes_ready(self):+ launcher = self.root / "launcher.cmd"+ launcher.write_bytes(f'@echo off\r\n"{sys.executable}" %*\r\n'.encode())+ host = runtime.Host("local", shutdown_timeout=2, startup_wait=20)+ plan, args, source, original = host._plan()+ with mock.patch.object(sys, "executable", str(launcher)):+ host._launch(args, source, original, plan["profile_revision"])+ try:+ result = host._wait_ready(30)+ self.assertTrue(result["ok"], result)+ self.assertTrue(result["generation_ready"])+ ready = json.loads(host.ready_path.read_text(encoding="utf-8"))+ self.assertNotEqual(ready["pid"], host.child.pid) # the server is below the launcher+ finally:+ host._stop_child()++ def test_a_ready_file_that_is_not_ours_is_named_in_the_status_phase(self):+ host = runtime.Host("local", shutdown_timeout=2, startup_wait=10)+ host.child = mock.Mock(pid=111)+ host.child.poll.return_value = None+ host.ready_path = self.root / "ready.json"+ host.ready_path.write_text(json.dumps({"pid": 222, "instance_id": "a" * 32}), encoding="utf-8")+ with mock.patch.object(runtime.procs, "parent_of", return_value=None):+ self.assertEqual(host._status()["phase"], "ready_file_not_matched")+ host.ready_path.unlink()+ self.assertEqual(host._status()["phase"], "loading")+def test_exclusive_lease_waits_for_shared_holders_and_shared_holders_coexist(self):first = runtime.acquire_activity_lease(shared=True)second = runtime.acquire_activity_lease(shared=True)diff --git a/tests/terminal/test_updater.py b/tests/terminal/test_updater.pyindex f8b6747c4..803bdba07 100644--- a/tests/terminal/test_updater.py+++ b/tests/terminal/test_updater.py@@ -180,6 +180,7 @@ class EngineUpdate(UpdaterCase):ran = (self.root / "ran.txt").read_text().splitlines()[0]self.assertIn(os.sep + ".s" + os.sep, ran)phases = [e.get("phase") for e in events if e.get("event") == "phase"]+ phases = [p for i, p in enumerate(phases) if i == 0 or p != "test" or phases[i - 1] != "test"] # progress lines of the test start repeat itself.assertEqual(phases, ["feed", "download", "verify", "verify", "test", "switch", "done"])progress = [e for e in events if e.get("event") == "progress"]self.assertTrue(progress and progress[-1]["percent"] == 100 and progress[-1]["done"] == progress[-1]["total"])@@ -457,7 +458,7 @@ class StartPathSignature(UpdaterCase):class LazyModel(UpdaterCase):"""A server that is "ready" with its model unloaded has not generated anything yet; the test start forces the load and the probe."""- def run_default(self, started, verified):+ def run_default(self, started, verified, tail=None):from strata_control import runtimeself.install_current()folder = fx.build_package(self.root / "lazy", "0.1.46", key=self.key)@@ -466,7 +467,8 @@ class LazyModel(UpdaterCase):shutil.copytree(folder, stage / "engine" / folder.name)component = ec.read_manifest(stage / "engine" / folder.name)with mock.patch.object(runtime, "start", return_value=started), mock.patch.object(runtime, "verify", return_value=verified) as verify, mock.patch.object(runtime, "stop", return_value={}):- result = updater.default_test_start(stage, component, self.ctx())+ with mock.patch.object(updater, "_log_tail", return_value=tail or ""):+ result = updater.default_test_start(stage, component, self.ctx())return result, verifydef test_an_unloaded_ready_server_is_made_to_answer_before_the_engine_passes(self):@@ -478,6 +480,51 @@ class LazyModel(UpdaterCase):def test_an_engine_that_cannot_answer_does_not_pass(self):with self.assertRaisesRegex(updater.TestStartFailed, "one-token probe"):self.run_default({"ok": True, "state": "ready", "generation_ready": False}, {"ok": False, "state": "failed", "error": "probe_generation_no_tokens"})+ def expired(self, phase, words, busy):+ # #3683: the engine log always mentions VRAM, so a wait that expired used to be blamed on a busy graphics card+ tail = " | strata serve: 358 MiB of VRAM free with everything loaded"+ with self.assertRaises(updater.TestStartFailed) as raised:+ self.run_default({"ok": False, "state": "starting", "error": "startup_wait_expired", "phase": phase}, {}, tail=tail)+ text = str(raised.exception)+ self.assertIn(words, text)+ self.assertEqual(raised.exception.busy, busy)+ self.assertNotIn("graphics card or memory looks busy", text)+ self.assertNotIn("358 MiB", text)+ self.assertIn("test-start-", text) # where the kept log is++ def test_an_expired_start_with_the_model_loaded_says_it_did_not_answer(self):+ self.expired("verifying_generation", "did not answer one token", False)++ def test_an_expired_start_with_an_unmatched_ready_file_says_so(self):+ self.expired("ready_file_not_matched", "could not match it to the engine process", False)++ def test_an_expired_start_that_was_still_loading_says_so(self):+ self.expired("loading", "still loading the model", True)++ def test_the_test_start_shows_progress_while_it_waits(self):+ self.install_current()+ folder = fx.build_package(self.root / "lazy", "0.1.46", key=self.key)+ stage = self.root / "stage"+ (stage / "engine").mkdir(parents=True)+ shutil.copytree(folder, stage / "engine" / folder.name)+ component = ec.read_manifest(stage / "engine" / folder.name)+ events = []+ ctx = self.ctx()+ ctx.emit = events.append++ def fake_start(name, **kw):+ for status in ({"state": "starting", "phase": "loading"}, {"state": "starting", "phase": "loading"},+ {"state": "starting", "phase": "verifying_generation"}, {"state": "ready"}):+ kw["on_status"](status)+ return {"ok": True, "state": "ready", "generation_ready": True}++ from strata_control import runtime+ with mock.patch.object(runtime, "start", side_effect=fake_start), mock.patch.object(runtime, "stop", return_value={}):+ updater.default_test_start(stage, component, ctx)+ lines = [e["message"] for e in events if e.get("phase") == "test"]+ self.assertEqual(len(lines), 2) # one per phase change, not one per poll+ self.assertIn("loading the model", lines[0])+ self.assertIn("answer one token", lines[1])class OtherComponents(UpdaterCase):