Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

updater: a failed test start keeps the runtime records, not only the engine log tail (#3654)

Niko1221bb62a048b89a
diff --git a/strata_control/updater.py b/strata_control/updater.py
index d5a7f7eb3..30f226ac3 100644
--- a/strata_control/updater.py
+++ b/strata_control/updater.py
@@ -792,6 +792,55 @@ def _keep_log(sandbox: Path, component: ec.EngineComponent, ctx: Context) -> Non
pass
+_RECORD_LIMIT = 65536 # bytes read from any one kept file
+_RECORDS_KEPT = 3 # failed test starts whose records stay in .updates/logs
+
+
+def _scrub(value, key: str = ""):
+ """A kept JSON record without anything that could authorise a request: keys that name a token or a key keep only their first
+ characters (enough to see that two records agree), the API key is dropped."""
+ if isinstance(value, dict):
+ return {k: _scrub(v, str(k)) for k, v in value.items()}
+ if isinstance(value, list):
+ return [_scrub(v, key) for v in value]
+ low = key.lower()
+ if isinstance(value, str) and value and ("api_key" in low or low in ("key", "authorization")):
+ return "<removed>"
+ if isinstance(value, str) and ("token" in low or "secret" in low or low.endswith("_key")):
+ return value[:8] + "...(rest removed)"
+ return value
+
+
+def _keep_records(sandbox: Path, component: ec.EngineComponent, ctx: Context) -> None:
+ """A failed test start keeps what the sandbox knew, because the sandbox is deleted right after (#3654): the run config the test used
+ (without keys), the host, child and ready records of the runtime (their PIDs show a mismatch), and the host and server logs, next to the
+ engine log in ``.updates/logs/test-start-<version>-failed``. The last few failed starts are kept. Best effort: it never hides the failure."""
+ try:
+ target = updates_dir(ctx.components) / "logs" / f"test-start-{component.version}-failed"
+ shutil.rmtree(target, ignore_errors=True)
+ target.mkdir(parents=True)
+ wanted = [sandbox / "run.json", sandbox / "engine.log"]
+ wanted += sorted((sandbox / "runtime").glob("*.json")) + sorted((sandbox / "runtime").glob("*.log"))
+ for source in wanted:
+ try:
+ if not source.is_file():
+ continue
+ data = source.read_bytes()[-_RECORD_LIMIT:]
+ if source.suffix == ".json":
+ try:
+ data = (json.dumps(_scrub(json.loads(data.decode("utf-8-sig"))), indent=1) + "\n").encode("utf-8")
+ except ValueError:
+ pass # not JSON after all: kept as it is
+ (target / source.name).write_bytes(data)
+ except OSError:
+ continue
+ folders = sorted((d for d in target.parent.glob("test-start-*-failed") if d.is_dir()), key=lambda d: d.stat().st_mtime)
+ for old in folders[:-_RECORDS_KEPT]:
+ shutil.rmtree(old, ignore_errors=True)
+ except OSError:
+ pass
+
+
def _warn_if_deep(ctx: Context, folder: Path, longest_inside: int) -> None:
"""Tell the person, in plain words, when the engine will sit too deep for Windows' ordinary path limit and long paths are
off. The install continues: Strata's own file work uses extended paths; only the engine's own loading may not."""
@@ -916,14 +965,18 @@ def default_test_start(stage_root: Path, component: ec.EngineComponent, ctx: Con
started = {**started, **runtime.verify(name, timeout=min(ctx.test_timeout, 600), load=True)}
if not started.get("ok") or started.get("state") != "ready" or started.get("generation_ready") is not True:
_keep_log(sandbox, component, ctx)
+ _keep_records(sandbox, component, ctx)
raise _test_start_failure(component, started, ctx, _log_tail(sandbox, component))
_keep_log(sandbox, component, ctx)
return {"mode": "full", "model": cfg.get("model_name") or source.stem, "state": started["state"],
"generation_ready": True, "detail": "started, answered a one-token probe, stopped"}
except TestStartFailed:
_keep_log(sandbox, component, ctx)
+ _keep_records(sandbox, component, ctx)
raise
except Exception as exc: # noqa: BLE001
+ _keep_log(sandbox, component, ctx)
+ _keep_records(sandbox, component, ctx)
raise TestStartFailed(component.version, f"{type(exc).__name__}: {exc}") from None
finally:
try:
diff --git a/tests/terminal/test_updater.py b/tests/terminal/test_updater.py
index 4969a34d4..267506a44 100644
--- a/tests/terminal/test_updater.py
+++ b/tests/terminal/test_updater.py
@@ -799,5 +799,64 @@ class AnotherCardFamilyNote(unittest.TestCase):
self.assertEqual(report["note"], "This build of Strata is for AMD RX 7000 cards. This computer has an AMD RX 9000 card (gfx1201). Ask for the RX 9000 build.")
+class FailedStartRecords(LazyModel):
+ """#3654: a failed test start used to leave only the last 64 KiB of the engine log, because the sandbox that held the host, child and
+ ready records was deleted at once. The records are kept (without tokens or keys) next to the log."""
+
+ def failing_start(self):
+ def start(name, **kw):
+ runtime_dir = Path(os.environ["STRATA_RUNTIME_DIR"]) # the sandbox's, set by default_test_start
+ runtime_dir.mkdir(parents=True, exist_ok=True)
+ (runtime_dir / f"{name}.json").write_text(json.dumps({"pid": 4321, "token": "a1b2c3d4" * 8, "state": "starting"}), encoding="utf-8")
+ (runtime_dir / f"{name}-{'a1b2c3d4' * 8}.child.json").write_text(json.dumps({"host_token": "a1b2c3d4" * 8, "child_pid": 111}), encoding="utf-8")
+ (runtime_dir / f"{name}-9f.ready.json").write_text(json.dumps({"pid": 222, "api_key": "sekret-key"}), encoding="utf-8")
+ (runtime_dir / f"{name}.server.log").write_text("server: model loaded\nserver: pid 222\n", encoding="utf-8")
+ (runtime_dir / f"{name}.host.log").write_text("", encoding="utf-8")
+ return {"ok": False, "state": "starting", "error": "startup_wait_expired", "phase": "ready_file_not_matched"}
+ return start
+
+ def test_the_runtime_records_of_a_failed_start_are_kept_without_tokens_or_keys(self):
+ from strata_control import runtime
+ self.install_current()
+ folder = fx.build_package(self.root / "lazy", "0.1.46", key=self.key)
+ stage = self.root / "stage"
+ (stage / "engine").mkdir(parents=True)
+ shutil.copytree(folder, stage / "engine" / folder.name)
+ component = ec.read_manifest(stage / "engine" / folder.name)
+ self.config.write_text(json.dumps({**json.loads(self.config.read_text()), "api_key": "sekret-key"}), encoding="utf-8")
+ with mock.patch.object(runtime, "start", side_effect=self.failing_start()), mock.patch.object(runtime, "stop", return_value={}):
+ with self.assertRaises(updater.TestStartFailed):
+ updater.default_test_start(stage, component, self.ctx())
+ kept = updater.updates_dir(self.components) / "logs" / f"test-start-{component.version}-failed"
+ names = sorted(p.name for p in kept.iterdir())
+ self.assertIn("run.json", names)
+ self.assertTrue(any(n.endswith(".child.json") for n in names), names)
+ self.assertTrue(any(n.endswith(".ready.json") for n in names), names)
+ self.assertIn("updtest.server.log", names)
+ text = "".join(p.read_text(encoding="utf-8") for p in kept.iterdir())
+ self.assertNotIn("sekret-key", text)
+ self.assertNotIn("a1b2c3d4a1b2c3d4", text, "no full token is kept")
+ self.assertIn('"child_pid": 111', text) # the PIDs that show a mismatch are
+ self.assertIn("server: pid 222", text)
+ self.assertFalse((self.components / ".updates").exists() and any(p.name.startswith("t-") for p in (self.components / ".updates").rglob("*") if p.is_dir()),
+ "the sandbox itself is still removed")
+
+ def test_only_the_last_few_failed_starts_are_kept(self):
+ logs = updater.updates_dir(self.components) / "logs"
+ for i in range(5):
+ folder = logs / f"test-start-0.1.4{i}-failed"
+ folder.mkdir(parents=True)
+ (folder / "run.json").write_text("{}", encoding="utf-8")
+ os.utime(folder, (1_700_000_000 + i, 1_700_000_000 + i))
+ from types import SimpleNamespace
+ sandbox = self.root / "sandbox"
+ sandbox.mkdir()
+ (sandbox / "run.json").write_text("{}", encoding="utf-8")
+ updater._keep_records(sandbox, SimpleNamespace(version="0.1.49"), self.ctx())
+ left = sorted(p.name for p in logs.glob("test-start-*-failed"))
+ self.assertEqual(len(left), 3, left)
+ self.assertIn("test-start-0.1.49-failed", left)
+
+
if __name__ == "__main__":
unittest.main()