Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Fix #3683: the test start no longer waits out its 10 minutes when a Windows venv launcher sits above the server

Niko122172d0bf667e50
diff --git a/strata_control/runtime.py b/strata_control/runtime.py
index 440504a65..2497957d9 100644
--- a/strata_control/runtime.py
+++ b/strata_control/runtime.py
@@ -468,6 +468,30 @@ def _engine_preflight(plan: dict) -> None:
raise ProfileError(str(exc), exc.exit_code) from None
+def _ready_pid_is_child(ready_pid, child_pid: int) -> bool:
+ """Is the process named in the ready file the one this host started?
+
+ Normally the pids are equal. A Windows venv (the python.exe of a source checkout's .venv) starts its real interpreter as a
+ CHILD of the launcher it was given, so the server that writes the ready file is one process below the pid ``Popen`` returned
+ (#3683: the staged engine was ready, this check never matched, and the start waited out its whole deadline). Only on Windows,
+ and only for a process that sits within three parents of the process this host started, so another process cannot pass."""
+ if type(ready_pid) is not int or ready_pid <= 0:
+ return False
+ if ready_pid == child_pid:
+ return True
+ if not WINDOWS:
+ return False
+ current = ready_pid
+ for _ in range(3):
+ parent = procs.parent_of(current)
+ if parent is None or parent <= 0:
+ return False
+ if parent == child_pid:
+ return True
+ current = parent
+ return False
+
+
def _check_start_candidate(name: str, expected_profile_revision: str | None,
expected_source_sha256: str | None) -> tuple[dict, str]:
plan, digest = _start_candidate(name)
@@ -481,9 +505,13 @@ def _check_start_candidate(name: str, expected_profile_revision: str | None,
def start(name: str, *, wait: float = 300, shutdown_timeout: float = 30,
expected_profile_revision: str | None = None,
expected_source_sha256: str | None = None,
- die_with_pid: int | None = None) -> dict:
+ die_with_pid: int | None = None, on_status=None) -> dict:
"""Start the saved profile under a new owner host.
+ ``on_status`` (optional) is called with each status answer while the start waits (about every 0.2 s; the caller throttles), so a
+ caller can show progress. The result of a wait that ran out names the last phase (``loading``, ``verifying_generation``,
+ ``ready_file_not_matched``).
+
The owner host is a background service: it is not tied to the terminal or the process that ran this command and
keeps serving after they end. It stops on `runtime stop`, when its server ends, or - only if `die_with_pid`
names a process - when that process is gone (the desktop app uses this for the engines it starts)."""
@@ -542,14 +570,20 @@ def start(name: str, *, wait: float = 300, shutdown_timeout: float = 30,
if wait == 0:
return {"ok": True, "profile": name, "state": "starting", "plan": plan}
deadline = time.monotonic() + wait
+ result: dict = {}
while time.monotonic() < deadline:
result = _rpc(name, "status")
+ if on_status is not None:
+ try:
+ on_status(result)
+ except Exception: # noqa: BLE001 - progress must never break a start
+ on_status = None
if result.get("state") in ("ready", "failed"):
return {**result, "ok": result["state"] == "ready",
"exit_code": 6 if result["state"] == "failed" else 0}
time.sleep(0.2)
return {"ok": False, "profile": name, "state": "starting", "error": "startup_wait_expired",
- "exit_code": 8}
+ "phase": result.get("phase"), "exit_code": 8}
def plan_stop(name: str) -> dict:
@@ -901,7 +935,7 @@ class Host:
if self.ready_path is not None:
try:
ready = json.loads(self.ready_path.read_text(encoding="utf-8"))
- if ready.get("pid") == child.pid and isinstance(ready.get("instance_id"), str):
+ if _ready_pid_is_child(ready.get("pid"), child.pid) and isinstance(ready.get("instance_id"), str):
key = (child.pid, ready["instance_id"])
if self.probe_key != key:
self.probe_key, self.probe_result = key, None
@@ -920,9 +954,13 @@ class Host:
return {**base, "state": "ready", "instance_id": ready["instance_id"],
"model_state": ready.get("model_state"),
**self.probe_result}
+ if isinstance(ready, dict):
+ # the server published its ready file but it is not the process this host started: say so, so a wait that
+ # ends can name it (before #3683 this looked exactly like a model that was still loading)
+ return {**base, "state": "starting", "phase": "ready_file_not_matched"}
except (OSError, ValueError):
pass
- return {**base, "state": "starting"}
+ return {**base, "state": "starting", "phase": "loading"}
def _wait_ready(self, timeout: float) -> dict:
deadline = time.monotonic() + timeout
@@ -1139,7 +1177,7 @@ class Host:
return {"ok": False, "state": current["state"], "error": "runtime_not_ready", "exit_code": 6}
try:
ready = json.loads(self.ready_path.read_text(encoding="utf-8"))
- if ready.get("pid") != self.child.pid or ready.get("instance_id") != current["instance_id"]:
+ if not _ready_pid_is_child(ready.get("pid"), self.child.pid) or ready.get("instance_id") != current["instance_id"]:
raise readiness.ReadinessError("probe_instance_mismatch")
host = self.args[self.args.index("--host") + 1]
result = readiness.verify(ready, self.source_bytes, expected_host=host,
diff --git a/strata_control/updater.py b/strata_control/updater.py
index 0f504ab73..7f4d719e9 100644
--- a/strata_control/updater.py
+++ b/strata_control/updater.py
@@ -76,12 +76,15 @@ class TestStartFailed(UpdateError):
BUSY = re.compile(r"timed?[ -]?out|out of memory|\boom\b|not enough (gpu |video |free )?memory|vram|memory allocation|"
r"cudaerrormemory|device (is )?(busy|in use)|resource busy|no free (gpu|memory)", re.I)
- def __init__(self, label: str, reason: str):
- self.busy = bool(self.BUSY.search(reason))
+ def __init__(self, label: str, reason: str, *, busy: bool | None = None, hint: str | None = None):
+ # busy: True/False says outright whether the card or memory was busy (the wait-expired reasons carry the engine's log tail, which
+ # mentions VRAM on every healthy start and must not be read as a busy card, #3683); None reads it from the words of the reason
+ self.busy = bool(self.BUSY.search(reason)) if busy is None else busy
+ if hint is None and self.busy:
+ hint = ("The graphics card or memory looks busy: close programs that use the card (games, video, other models) "
+ "and run `strata update` again.")
super().__init__(f"The new engine {label} did not pass its test start ({reason}). It was not installed; "
- "your current engine is unchanged."
- + (" The graphics card or memory looks busy: close programs that use the card (games, video, other models) "
- "and run `strata update` again." if self.busy else ""), 6)
+ "your current engine is unchanged." + (" " + hint if hint else ""), 6)
self.label = label
self.reason = reason
@@ -702,6 +705,48 @@ def _warn_if_deep(ctx: Context, folder: Path, longest_inside: int) -> None:
ctx.say("warn", words, component="engine")
+def _test_progress(ctx: Context, component: ec.EngineComponent):
+ """What the person sees while the staged engine is test-started: a line when the phase changes and one about every 30 seconds,
+ so a long load or a stuck probe is never a silent wait (#3683: 10 minutes of nothing, then a timeout)."""
+ began = time.monotonic()
+ seen = {"phase": None, "at": 0.0}
+ words = {"loading": "is loading the model",
+ "verifying_generation": "has loaded the model; waiting for it to answer one token",
+ "ready_file_not_matched": "says it is ready; checking that it is the engine Strata started"}
+
+ def progress(status: dict) -> None:
+ phase = status.get("phase") if status.get("state") == "starting" else None
+ if phase not in words:
+ return
+ now = time.monotonic()
+ if phase != seen["phase"] or now - seen["at"] >= 30:
+ seen["phase"], seen["at"] = phase, now
+ ctx.say("test", f"Engine {component.version} {words[phase]} ({int(now - began)} s)", component="engine")
+
+ return progress
+
+
+def _test_start_failure(component: ec.EngineComponent, started: dict, ctx: Context, tail: str) -> TestStartFailed:
+ """The plain reason a test start did not pass, in the words of what the engine was doing when it ran out of time."""
+ error = started.get("error")
+ minutes = max(1, int(min(ctx.test_timeout, 600) // 60))
+ log = updates_dir(ctx.components) / "logs" / f"test-start-{component.version}.log"
+ if error == "startup_wait_expired":
+ phase = started.get("phase")
+ kept = f" The engine's log from this test is in {log}."
+ if phase == "verifying_generation":
+ return TestStartFailed(component.version, f"it loaded the model but did not answer one token within {minutes} minutes",
+ busy=False, hint="Nothing was changed. Run `strata update` again; if it happens again, report it with the log." + kept)
+ if phase == "ready_file_not_matched":
+ return TestStartFailed(component.version, f"it reported that it was ready, but Strata could not match it to the engine process it "
+ f"started", busy=False,
+ hint="This looks like a problem in Strata, not in your engine or card. Nothing was changed; report it with the log." + kept)
+ return TestStartFailed(component.version, f"it was still loading the model after {minutes} minutes", busy=True,
+ hint="Nothing was changed. A slow disk or a card that other programs are using can do this: close them and run "
+ "`strata update` again." + kept)
+ return TestStartFailed(component.version, f"it did not answer a one-token probe ({error or started.get('state')}){tail}")
+
+
def default_test_start(stage_root: Path, component: ec.EngineComponent, ctx: Context) -> dict:
"""Start the staged engine through the real start path (owner runtime, `component:strata-engine`, a saved model), wait for
the one-token generation probe, and stop it. Everything runs in an isolated runtime, profile and component folder, so a
@@ -732,15 +777,16 @@ def default_test_start(stage_root: Path, component: ec.EngineComponent, ctx: Con
profiles.create(name, str(config_path))
started = None
try:
- started = runtime.start(name, wait=min(ctx.test_timeout, 600), shutdown_timeout=20)
+ started = runtime.start(name, wait=min(ctx.test_timeout, 600), shutdown_timeout=20,
+ on_status=_test_progress(ctx, component))
if started.get("ok") and started.get("state") == "ready" and started.get("generation_ready") is not True:
# a server that loads its model on the first request is "ready" without having generated anything:
# force the load and the one-token probe, because that is what the new engine has to be able to do
+ ctx.say("test", f"Engine {component.version} is up; asking it for one token", component="engine")
started = {**started, **runtime.verify(name, timeout=min(ctx.test_timeout, 600), load=True)}
if not started.get("ok") or started.get("state") != "ready" or started.get("generation_ready") is not True:
- raise TestStartFailed(component.version,
- f"it did not answer a one-token probe ({started.get('error') or started.get('state')})"
- + _log_tail(sandbox, component))
+ _keep_log(sandbox, component, ctx)
+ raise _test_start_failure(component, started, ctx, _log_tail(sandbox, component))
_keep_log(sandbox, component, ctx)
return {"mode": "full", "model": cfg.get("model_name") or source.stem, "state": started["state"],
"generation_ready": True, "detail": "started, answered a one-token probe, stopped"}
diff --git a/tests/terminal/test_runtime.py b/tests/terminal/test_runtime.py
index c54b9c6a8..7b795b9bc 100644
--- a/tests/terminal/test_runtime.py
+++ b/tests/terminal/test_runtime.py
@@ -1195,6 +1195,53 @@ class RuntimeTests(unittest.TestCase):
with self.assertRaisesRegex(profiles.ProfileError, "Everyone"):
runtime.runtime_dir()
+ def test_ready_pid_may_be_a_child_of_the_launcher_on_windows_only(self):
+ # #3683: a Windows venv's python.exe starts the real interpreter as its child, so the server that writes the ready file
+ # is one process below the pid the host started. It must still match; a stranger or a distant process must not.
+ parents = {200: 100, 300: 200, 400: 300, 500: 400}
+ with mock.patch.object(runtime.procs, "parent_of", side_effect=lambda pid: parents.get(pid)):
+ with mock.patch.object(runtime, "WINDOWS", True):
+ self.assertTrue(runtime._ready_pid_is_child(100, 100))
+ self.assertTrue(runtime._ready_pid_is_child(200, 100))
+ self.assertTrue(runtime._ready_pid_is_child(300, 100))
+ self.assertTrue(runtime._ready_pid_is_child(400, 100))
+ self.assertFalse(runtime._ready_pid_is_child(500, 100)) # too far below
+ self.assertFalse(runtime._ready_pid_is_child(999, 100)) # unrelated
+ self.assertFalse(runtime._ready_pid_is_child(None, 100))
+ self.assertFalse(runtime._ready_pid_is_child("200", 100))
+ self.assertFalse(runtime._ready_pid_is_child(0, 100))
+ with mock.patch.object(runtime, "WINDOWS", False):
+ self.assertTrue(runtime._ready_pid_is_child(100, 100))
+ self.assertFalse(runtime._ready_pid_is_child(200, 100)) # POSIX has no launcher in between
+
+ @unittest.skipUnless(WINDOWS, "the venv launcher is a Windows thing")
+ def test_server_started_through_a_launcher_process_becomes_ready(self):
+ launcher = self.root / "launcher.cmd"
+ launcher.write_bytes(f'@echo off\r\n"{sys.executable}" %*\r\n'.encode())
+ host = runtime.Host("local", shutdown_timeout=2, startup_wait=20)
+ plan, args, source, original = host._plan()
+ with mock.patch.object(sys, "executable", str(launcher)):
+ host._launch(args, source, original, plan["profile_revision"])
+ try:
+ result = host._wait_ready(30)
+ self.assertTrue(result["ok"], result)
+ self.assertTrue(result["generation_ready"])
+ ready = json.loads(host.ready_path.read_text(encoding="utf-8"))
+ self.assertNotEqual(ready["pid"], host.child.pid) # the server is below the launcher
+ finally:
+ host._stop_child()
+
+ def test_a_ready_file_that_is_not_ours_is_named_in_the_status_phase(self):
+ host = runtime.Host("local", shutdown_timeout=2, startup_wait=10)
+ host.child = mock.Mock(pid=111)
+ host.child.poll.return_value = None
+ host.ready_path = self.root / "ready.json"
+ host.ready_path.write_text(json.dumps({"pid": 222, "instance_id": "a" * 32}), encoding="utf-8")
+ with mock.patch.object(runtime.procs, "parent_of", return_value=None):
+ self.assertEqual(host._status()["phase"], "ready_file_not_matched")
+ host.ready_path.unlink()
+ self.assertEqual(host._status()["phase"], "loading")
+
def test_exclusive_lease_waits_for_shared_holders_and_shared_holders_coexist(self):
first = runtime.acquire_activity_lease(shared=True)
second = runtime.acquire_activity_lease(shared=True)
diff --git a/tests/terminal/test_updater.py b/tests/terminal/test_updater.py
index f8b6747c4..803bdba07 100644
--- a/tests/terminal/test_updater.py
+++ b/tests/terminal/test_updater.py
@@ -180,6 +180,7 @@ class EngineUpdate(UpdaterCase):
ran = (self.root / "ran.txt").read_text().splitlines()[0]
self.assertIn(os.sep + ".s" + os.sep, ran)
phases = [e.get("phase") for e in events if e.get("event") == "phase"]
+ phases = [p for i, p in enumerate(phases) if i == 0 or p != "test" or phases[i - 1] != "test"] # progress lines of the test start repeat it
self.assertEqual(phases, ["feed", "download", "verify", "verify", "test", "switch", "done"])
progress = [e for e in events if e.get("event") == "progress"]
self.assertTrue(progress and progress[-1]["percent"] == 100 and progress[-1]["done"] == progress[-1]["total"])
@@ -457,7 +458,7 @@ class StartPathSignature(UpdaterCase):
class LazyModel(UpdaterCase):
"""A server that is "ready" with its model unloaded has not generated anything yet; the test start forces the load and the probe."""
- def run_default(self, started, verified):
+ def run_default(self, started, verified, tail=None):
from strata_control import runtime
self.install_current()
folder = fx.build_package(self.root / "lazy", "0.1.46", key=self.key)
@@ -466,7 +467,8 @@ class LazyModel(UpdaterCase):
shutil.copytree(folder, stage / "engine" / folder.name)
component = ec.read_manifest(stage / "engine" / folder.name)
with mock.patch.object(runtime, "start", return_value=started), mock.patch.object(runtime, "verify", return_value=verified) as verify, mock.patch.object(runtime, "stop", return_value={}):
- result = updater.default_test_start(stage, component, self.ctx())
+ with mock.patch.object(updater, "_log_tail", return_value=tail or ""):
+ result = updater.default_test_start(stage, component, self.ctx())
return result, verify
def test_an_unloaded_ready_server_is_made_to_answer_before_the_engine_passes(self):
@@ -478,6 +480,51 @@ class LazyModel(UpdaterCase):
def test_an_engine_that_cannot_answer_does_not_pass(self):
with self.assertRaisesRegex(updater.TestStartFailed, "one-token probe"):
self.run_default({"ok": True, "state": "ready", "generation_ready": False}, {"ok": False, "state": "failed", "error": "probe_generation_no_tokens"})
+ def expired(self, phase, words, busy):
+ # #3683: the engine log always mentions VRAM, so a wait that expired used to be blamed on a busy graphics card
+ tail = " | strata serve: 358 MiB of VRAM free with everything loaded"
+ with self.assertRaises(updater.TestStartFailed) as raised:
+ self.run_default({"ok": False, "state": "starting", "error": "startup_wait_expired", "phase": phase}, {}, tail=tail)
+ text = str(raised.exception)
+ self.assertIn(words, text)
+ self.assertEqual(raised.exception.busy, busy)
+ self.assertNotIn("graphics card or memory looks busy", text)
+ self.assertNotIn("358 MiB", text)
+ self.assertIn("test-start-", text) # where the kept log is
+
+ def test_an_expired_start_with_the_model_loaded_says_it_did_not_answer(self):
+ self.expired("verifying_generation", "did not answer one token", False)
+
+ def test_an_expired_start_with_an_unmatched_ready_file_says_so(self):
+ self.expired("ready_file_not_matched", "could not match it to the engine process", False)
+
+ def test_an_expired_start_that_was_still_loading_says_so(self):
+ self.expired("loading", "still loading the model", True)
+
+ def test_the_test_start_shows_progress_while_it_waits(self):
+ self.install_current()
+ folder = fx.build_package(self.root / "lazy", "0.1.46", key=self.key)
+ stage = self.root / "stage"
+ (stage / "engine").mkdir(parents=True)
+ shutil.copytree(folder, stage / "engine" / folder.name)
+ component = ec.read_manifest(stage / "engine" / folder.name)
+ events = []
+ ctx = self.ctx()
+ ctx.emit = events.append
+
+ def fake_start(name, **kw):
+ for status in ({"state": "starting", "phase": "loading"}, {"state": "starting", "phase": "loading"},
+ {"state": "starting", "phase": "verifying_generation"}, {"state": "ready"}):
+ kw["on_status"](status)
+ return {"ok": True, "state": "ready", "generation_ready": True}
+
+ from strata_control import runtime
+ with mock.patch.object(runtime, "start", side_effect=fake_start), mock.patch.object(runtime, "stop", return_value={}):
+ updater.default_test_start(stage, component, ctx)
+ lines = [e["message"] for e in events if e.get("phase") == "test"]
+ self.assertEqual(len(lines), 2) # one per phase change, not one per poll
+ self.assertIn("loading the model", lines[0])
+ self.assertIn("answer one token", lines[1])
class OtherComponents(UpdaterCase):