Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(agent-runtimes): safe OpenClaw upgrades, real start-up readiness, no Temp Hermes (#1019)
Ruben Lütke1c2d3129547c
diff --git a/docs/agent-runtimes.md b/docs/agent-runtimes.mdindex 4742a2c881..b16678e844 100644--- a/docs/agent-runtimes.md+++ b/docs/agent-runtimes.md@@ -338,6 +338,13 @@ replayed during `session/load` is swallowed because the chat already shows it.`agent_runtimes` folder) once per app run. An agent's old folder is moved:its session store is copied into the profile, then the old folder and itsprivate environment are removed.+- **Which `hermes` runs:** the official installer's own location answers first+ (`%LOCALAPPDATA%\hermes\bin\hermes.exe` on Windows, `~/.local/bin/hermes`+ or `~/.hermes/bin/hermes` elsewhere), PATH only after it. A PATH entry inside+ a Dodaios agent folder or inside a temporary folder (a test or audit run's+ throwaway install, e.g. `...\Temp\jarvis-hermes-audit-*\runtimes\hermes\bin`)+ never answers; the user's PATH itself is left as it is+ (`hermes._binary`).- **Config Dodaios writes** (`config.yaml`): `_config_version` from the pins (soHermes migrates the file forward), `model.provider` / `model.default` /`model.base_url`, `terminal.cwd` = the agent workspace, Hermes' own memory@@ -476,6 +483,22 @@ replayed during `session/load` is swallowed because the chat already shows it.file). A Gateway exit code 78 (invalid config) runs`openclaw doctor --fix --yes --non-interactive` once (contained, killedafter two minutes), then retries.+- **Upgrades and start-up admission:** `/healthz` answers before the Gateway+ finished inspecting the agent's database (2026.10+), and a message sent+ meanwhile is refused with UNAVAILABLE "has not completed startup inspection+ and preparation". A started Gateway is therefore handed to a turn only once+ its own log says the database was admitted (at most two minutes; a release+ that logs no admission is ready at `/healthz`). A database an older release+ wrote ("uses schema version N; run openclaw doctor --fix") gets OpenClaw's+ own migration: the Gateway is stopped, `openclaw doctor --fix` runs on the+ agent's state folder (output in `doctor.log`, at most five minutes), and the+ Gateway starts again. The same migration runs before the start when the+ installed release differs from the one the folder last ran+ (`runtime-build.txt`). A database still refused afterwards fails the start+ with a readable reason in the chat. A turn whose message is refused as+ "not started yet" sends it again with a growing pause for up to three+ minutes on the same Gateway; a refusal that needs the repair gets one+ Gateway restart. Neither refusal reaches a model.- **Dodaios tools:** OpenClaw rejects per-session `mcpServers`, so the Dodaiosserver is configured in the agent's own state directory. Values such as`${JARVIS_CONTROL_API_KEY}` are substituted from the Gateway's environment,diff --git a/dodaios/agent_chat/runner_acp.py b/dodaios/agent_chat/runner_acp.pyindex 0e7c8cde72..15b719dcd8 100644--- a/dodaios/agent_chat/runner_acp.py+++ b/dodaios/agent_chat/runner_acp.py@@ -19,7 +19,7 @@ from typing import Anyfrom dodaios.agent_runtimes import RUNNER_RUNTIMES, driver, gatewayfrom dodaios.agent_runtimes.acp import AcpTurn-from dodaios.agent_runtimes.base import RuntimeTurn, RuntimeUnavailable+from dodaios.agent_runtimes.base import RuntimeTurn, RuntimeUnavailable, not_ready_kindfrom dodaios.agent_runtimes.model_map import RouteUnavailable, prepare_routelog = logging.getLogger(__name__)@@ -319,12 +319,14 @@ async def plan_runtime_turn(if route.api_key:gateway.unwatch_failure(route.api_key, failure)acp = plan.acp if plan is not None else None+ # A runtime still finishing its start keeps its process: a restart+ # would only begin that start again (the turn sends once more).+ starting = acp is not None and not_ready_kind(acp.error) == "starting"if launch.invalidate is not None and (handle.cancel.is_set()or (failure.done() and not failure.cancelled())or acp is None- or not acp.saw_result- or acp.status != "done"+ or ((not acp.saw_result or acp.status != "done") and not starting)):launch.invalidate()release_launch()diff --git a/dodaios/agent_chat/runner_cli.py b/dodaios/agent_chat/runner_cli.pyindex dd4e463d7c..2f9a364eae 100644--- a/dodaios/agent_chat/runner_cli.py+++ b/dodaios/agent_chat/runner_cli.py@@ -63,9 +63,9 @@ import osimport shutilimport timeimport uuid-from collections.abc import Callable, Iterable+from collections.abc import Awaitable, Callable, Iterablefrom concurrent.futures import Future-from contextlib import contextmanager+from contextlib import contextmanager, suppressfrom contextvars import ContextVarfrom dataclasses import dataclass, field, replacefrom pathlib import Path@@ -3116,6 +3116,71 @@ def _resume_was_lost(error: str | None, runner: str = "") -> bool:return any(m in low for m in _RESUME_LOST_MARKERS)+#: How long a turn keeps sending its message again while an agent runtime is+#: still starting (or repairing its data) before the refusal stands.+_RUNTIME_READY_WAIT_S: Final[float] = 180.0+#: First pause between two sends, doubled up to the cap.+_RUNTIME_READY_FIRST_DELAY_S: Final[float] = 1.0+_RUNTIME_READY_MAX_DELAY_S: Final[float] = 10.0+++async def _send_again_when_runtime_ready(+ handle: TurnHandle,+ outcome: _Outcome,+ runner: str,+ send: Callable[[], Awaitable[_Outcome]],+ recovery: Any,+) -> _Outcome:+ """Send the message again while an agent runtime refuses it as not ready yet.++ OpenClaw answers a message that arrives while its Gateway finishes its+ start-up inspection with UNAVAILABLE before any model call, and one whose+ stored data needs a migration until its restart ran it. The person's+ message is never dropped for that: it is sent again with a growing pause+ (the failed turn already marked a Gateway that needs repair for a restart)+ for at most :data:`_RUNTIME_READY_WAIT_S`. Nothing was billed by a refusal.+ """+ from dodaios.agent_chat.runner_acp import supports_runner+ from dodaios.agent_runtimes.base import not_ready_kind++ if not supports_runner(runner):+ return outcome+ loop = asyncio.get_running_loop()+ deadline = loop.time() + _RUNTIME_READY_WAIT_S+ delay = _RUNTIME_READY_FIRST_DELAY_S+ repairs = 0+ while (+ outcome.status == "error"+ and not recovery.had_tool_calls+ and not handle.cancel.is_set()+ ):+ kind = not_ready_kind(outcome.error)+ if not kind:+ break+ if kind == "repair":+ if repairs:+ break # one restart (with its migration) per turn; its error stands+ repairs += 1+ left = deadline - loop.time()+ if left <= 0:+ log.warning(+ "agent chat %s: %s still not ready after %.0f s",+ handle.turn_id, runner, _RUNTIME_READY_WAIT_S,+ )+ break+ log.info(+ "agent chat %s: %s not ready (%s: %s); sending again in %.0f s",+ handle.turn_id, runner, kind, (outcome.error or "")[:160], min(delay, left),+ )+ with suppress(TimeoutError): # the pause ended; send again below+ await asyncio.wait_for(handle.cancel.wait(), timeout=min(delay, left))+ if handle.cancel.is_set():+ break+ delay = min(delay * 2, _RUNTIME_READY_MAX_DELAY_S)+ outcome = await send()+ return outcome++# A print-mode CLI that cannot ask back treats an unanswered permission# prompt as "the user cancelled this tool" and then *exits* — Grok# StopCancelled / agy 1.1.26. The model never gets a second round. These@@ -3345,6 +3410,17 @@ async def run_cli_turn(outcome = await _run_cli_once(handle, user_text, runner, None, identity=ident, bridge=bridge)+ resume = None+ if outcome.status == "error" and not recovery.had_tool_calls:+ outcome = await _send_again_when_runtime_ready(+ handle,+ outcome,+ runner,+ lambda: _run_cli_once(+ handle, user_text, runner, resume, identity=ident, bridge=bridge+ ),+ recovery,+ )receipt_hint = recovery.hint() if identity else Noneif ((diff --git a/dodaios/agent_runtimes/base.py b/dodaios/agent_runtimes/base.pyindex 97e5c98f27..e25484bd15 100644--- a/dodaios/agent_runtimes/base.py+++ b/dodaios/agent_runtimes/base.py@@ -100,6 +100,39 @@ class RuntimeUnavailable(Exception):"""The runtime cannot run this turn; the message is shown in the chat."""+#: A runtime still finishing its start refuses a message before any model call+#: (OpenClaw 2026.10: UNAVAILABLE "Agent main has not completed startup+#: inspection and preparation"). The same process admits it moments later.+_STARTING_MARKERS: Final[tuple[str, ...]] = (+ "has not completed startup inspection",+ "startup preparation still running",+)++#: A runtime whose stored data an older release wrote refuses every message+#: until its own migration ran; a restart of its process runs it+#: (``openclaw._admitted``).+_REPAIR_MARKERS: Final[tuple[str, ...]] = (+ "uses schema version",+ "session stores for agent",+ "agentdatabaseadmissionerror",+)+++def not_ready_kind(error: str | None) -> str:+ """Why a runtime refused a message before any model call.++ ``starting`` (send it again to the same process shortly), ``repair``+ (restart the process: its start migrates the data) or ``""`` (any other+ error, never retried here).+ """+ low = (error or "").lower()+ if any(marker in low for marker in _STARTING_MARKERS):+ return "starting"+ if any(marker in low for marker in _REPAIR_MARKERS):+ return "repair"+ return ""++@dataclass(frozen=True, slots=True)class RuntimeStatus:"""What ``GET /api/agent-runtimes`` reports for one runtime."""diff --git a/dodaios/agent_runtimes/hermes.py b/dodaios/agent_runtimes/hermes.pyindex 49ad32f243..889be0cd2c 100644--- a/dodaios/agent_runtimes/hermes.py+++ b/dodaios/agent_runtimes/hermes.py@@ -116,28 +116,83 @@ _BIN_BLOCKER_TEXT: Final[str] = (def _binary() -> str | None:- """The person's Hermes launcher, never one inside a Jarvis agent folder.-- Builds before the profile layout left per-agent ``bin`` folders on the- user's PATH (``path_cleanup`` removes them); a stale one must not shadow- the real install.+ """The Hermes launcher: the official installer's location first, then PATH.++ The setup job installs Hermes with its official installer, which puts the+ launcher in a fixed per-user folder (:func:`_install_locations`); that one+ answers before anything on PATH. A PATH entry never answers when it is+ inside a Jarvis agent folder (builds before the profile layout left+ per-agent ``bin`` folders on the user's PATH; ``path_cleanup`` removes+ them) or inside a temporary folder (a test or audit run that installed a+ throwaway Hermes and put its ``bin`` on the user's PATH; the PATH itself+ is left as it is)."""- found = which("hermes", "hermes.exe", "hermes.cmd", skip=_in_jarvis_data)- if found:- return found+ found_path = first_existing(_install_locations())+ if found_path is not None and not _never_on_path(str(found_path.parent)):+ return str(found_path)+ return which("hermes", "hermes.exe", "hermes.cmd", skip=_never_on_path)+++def _install_locations() -> list[Path]:+ """Where Hermes' official installers put the launcher, per OS."""+ if is_windows():+ local = os.environ.get("LOCALAPPDATA")+ if not local:+ return []+ # The installer swaps ``hermes.exe`` during an update (it leaves a+ # ``hermes.exe.deleteme`` behind); its ``hermes.cmd`` stays meanwhile.+ bin_dir = Path(local) / "hermes" / "bin"+ return [bin_dir / "hermes.exe", bin_dir / "hermes.cmd"]home = Path.home()- candidates = [home / ".local" / "bin" / "hermes", home / ".hermes" / "bin" / "hermes"]- local = os.environ.get("LOCALAPPDATA")- if local:- candidates.insert(0, Path(local) / "hermes" / "bin" / "hermes.exe")- found_path = first_existing(candidates)- return str(found_path) if found_path else None+ return [home / ".local" / "bin" / "hermes", home / ".hermes" / "bin" / "hermes"]+++def _temp_roots() -> list[str]:+ """This machine's temporary folders, normalised for a prefix check."""+ import tempfile++ roots = {tempfile.gettempdir(), os.environ.get("TEMP", ""), os.environ.get("TMP", "")}+ if not is_windows():+ # Compared against, never written to.+ roots.update({"/tmp", "/var/tmp", os.environ.get("TMPDIR", "")}) # noqa: S108+ out: list[str] = []+ for root in roots:+ if not root:+ continue+ out.append(_norm(root))+ with contextlib.suppress(OSError): # an unresolvable root still counts as written+ out.append(_norm(str(Path(root).resolve())))+ return sorted(set(out))+++def _norm(path: str) -> str:+ return os.path.normcase(os.path.normpath(os.path.expandvars(path.strip().strip('"'))))+++#: A Windows temporary folder under any spelling of the profile path+#: (``C:\Users\ADMINI~1\AppData\Local\Temp`` and the long name alike).+_WINDOWS_TEMP: Final[re.Pattern[str]] = re.compile(+ r"[\\/]appdata[\\/]local[\\/]temp(?:[\\/]|$)", re.IGNORECASE+)+++def _in_temp(directory: str) -> bool:+ if not directory.strip():+ return False+ normal = _norm(directory)+ if _WINDOWS_TEMP.search(normal):+ return True+ for root in _temp_roots():+ if normal == root or normal.startswith(root.rstrip("\\/") + os.sep):+ return True+ return False-def _in_jarvis_data(directory: str) -> bool:+def _never_on_path(directory: str) -> bool:+ """A PATH entry whose ``hermes`` must never run an agent."""from dodaios.agent_runtimes.path_cleanup import is_jarvis_hermes_bin- return is_jarvis_hermes_bin(directory)+ return is_jarvis_hermes_bin(directory) or _in_temp(directory)def _install_hint() -> str:diff --git a/dodaios/agent_runtimes/openclaw.py b/dodaios/agent_runtimes/openclaw.pyindex 9277c88cfc..c56ad2e20d 100644--- a/dodaios/agent_runtimes/openclaw.py+++ b/dodaios/agent_runtimes/openclaw.py@@ -94,9 +94,35 @@ _WARM_SLOT_WAIT_S: Final[float] = 0.5#: How long ``openclaw doctor --fix`` may run before it is stopped._DOCTOR_TIMEOUT_S: Final[float] = 120.0+#: How long the migration pass (``openclaw doctor --fix`` after an update) may+#: run: it rewrites the agent's database, which takes longer than a config fix.+_MIGRATE_TIMEOUT_S: Final[float] = 300.0+#: ``EX_CONFIG``: the Gateway refused its configuration._EXIT_CONFIG: Final[int] = 78+#: How long a started Gateway may take to finish its agent database admission+#: after ``/healthz`` already answers (13-45 s measured on Windows, 2026.10.8).+_ADMISSION_WAIT_S: Final[float] = 120.0++#: What the Gateway logs about its agent database admission (2026.10+). A+#: release that logs none of these is ready once ``/healthz`` answers.+_ADMISSION_PENDING: Final[tuple[str, ...]] = (+ "has not completed startup inspection and preparation",+)+_ADMISSION_RECOVERED: Final[tuple[str, ...]] = (+ "agent database recovered after background inspection",+)+#: The database was written by an older release and needs ``doctor --fix``.+_ADMISSION_DEGRADED: Final[re.Pattern[str]] = re.compile(+ r"agent database remains degraded|agent database \S.{0,400}?uses schema version \d+",+ re.IGNORECASE,+)+_DEGRADED_REASON: Final[re.Pattern[str]] = re.compile(r'reason="([^"]+)"')++#: The release the agent's state folder last ran under (``--version`` line).+_BUILD_MARKER: Final[str] = "runtime-build.txt"+#: Node.js for Jarvis' private copy (the release OpenClaw's own user-space#: installer pins; inside the engine range below). OpenClaw's live#: ``install.ps1`` refuses a private Node older than 24.22.1 / 26.12.1 since@@ -967,6 +993,15 @@ class OpenClawRuntime:)try:await asyncio.to_thread(_reap_stale_gateway, home)+ build = await asyncio.to_thread(self._installed_build)+ if await asyncio.to_thread(_upgraded_since_last_start, home, build):+ # A new release may need the agent's database migrated; the+ # Gateway refuses every message until it is (2026.10.8: "uses+ # schema version 24; run openclaw doctor --fix"). No Gateway of+ # this folder runs now: the migration has the database alone.+ log.info("agent runtimes: OpenClaw changed for %s (%s); migrating its data",+ key, build)+ await _migrate(launcher, env, home)gateway = await self._spawn_until_ready(key, home, port, env_hash, launcher, env, lock)if gateway is None:# The port was taken before the Gateway bound it: once more on a@@ -978,11 +1013,66 @@ class OpenClawRuntime:key, home, port, env_hash, launcher, env, lock, last_try=True)assert gateway is not None+ gateway = await self._admitted(gateway, env_hash, launcher, env, lock)+ await asyncio.to_thread(_remember_build, home, build)return gatewayexcept BaseException:lock.release()raise+ def _installed_build(self) -> str:+ """The installed release's ``--version`` line ("" when unknown)."""+ status = self.detect(fast=True)+ return status.build or status.version++ async def _admitted(+ self,+ gateway: _Gateway,+ env_hash: str,+ launcher: list[str],+ env: dict[str, str],+ lock: HomeFileLock,+ ) -> _Gateway:+ """``gateway`` once its agent database admits messages.++ ``/healthz`` answers before the Gateway finished inspecting the agent's+ database; a message sent meanwhile is refused (UNAVAILABLE "has not+ completed startup inspection and preparation"). A database an older+ release wrote stays refused until ``openclaw doctor --fix`` migrated+ it: that runs once, with the Gateway stopped, then the Gateway starts+ again. A database still refused after it fails the start with the+ reason, never a turn that silently goes nowhere.+ """+ log_path = gateway.home / "gateway.log"+ try:+ state = await _await_admission(log_path, gateway.proc)+ if state != "degraded":+ return gateway+ reason = _degraded_reason(log_path)+ log.warning("agent runtimes: OpenClaw data of %s needs a migration: %s",+ gateway.key, reason)+ await self._stop_gateway(gateway, keep_lock=True)+ migrated = await _migrate(launcher, env, gateway.home)+ gateway = await self._spawn_until_ready(+ gateway.key, gateway.home, gateway.port, env_hash, launcher, env, lock,+ last_try=True,+ )+ assert gateway is not None+ state = await _await_admission(log_path, gateway.proc)+ except BaseException:+ # Never a Gateway left behind with the key in its environment.+ if gateway.alive():+ await asyncio.shield(self._stop_gateway(gateway, keep_lock=True))+ raise+ if state != "degraded":+ log.info("agent runtimes: OpenClaw data of %s migrated", gateway.key)+ return gateway+ await self._stop_gateway(gateway, keep_lock=True)+ reason = _degraded_reason(log_path) or reason+ log.error("agent runtimes: OpenClaw data of %s still needs repair after doctor --fix: %s",+ gateway.key, reason)+ raise RuntimeUnavailable(_repair_failed_message(reason, migrated))+async def _spawn_until_ready(self,key: str,@@ -1291,13 +1381,24 @@ def _gateway_token(home: Path) -> str:return token-async def _run_doctor(launcher: list[str], env: dict[str, str], home: Path) -> None:+async def _run_doctor(+ launcher: list[str], env: dict[str, str], home: Path, *, timeout_s: float | None = None+) -> int | None:+ """``openclaw doctor --fix`` on the agent's state folder, non-interactive.++ ``env`` carries the folder's ``OPENCLAW_STATE_DIR`` / ``OPENCLAW_CONFIG_PATH``,+ so the doctor repairs this agent's config and database and nothing of the+ person's own OpenClaw. Its output goes to ``doctor.log`` in the folder.+ Returns its exit code (``None`` = it could not start or was stopped).+ """from dodaios.core.process_tree import make_process_treefrom dodaios.core.process_utils import NO_WINDOW_CREATIONFLAGStree = make_process_tree("openclaw-doctor")proc: asyncio.subprocess.Process | None = None+ handle = Nonetry:+ handle = (home / "doctor.log").open("wb")proc = await asyncio.create_subprocess_exec(*launcher,"doctor",@@ -1307,21 +1408,136 @@ async def _run_doctor(launcher: list[str], env: dict[str, str], home: Path) -> Ncwd=str(home),env=env,stdin=asyncio.subprocess.DEVNULL,- stdout=asyncio.subprocess.DEVNULL,- stderr=asyncio.subprocess.DEVNULL,+ stdout=handle,+ stderr=handle,creationflags=NO_WINDOW_CREATIONFLAGS,start_new_session=os.name != "nt",)tree.assign(proc.pid)- await asyncio.wait_for(proc.wait(), timeout=_DOCTOR_TIMEOUT_S)+ return await asyncio.wait_for(+ proc.wait(), timeout=_DOCTOR_TIMEOUT_S if timeout_s is None else timeout_s+ )except (OSError, TimeoutError) as exc:log.warning("agent runtimes: openclaw doctor --fix failed: %s", exc)if proc is not None and proc.returncode is None:with contextlib.suppress(ProcessLookupError, OSError):proc.kill()+ return Nonefinally:# Never leave a doctor running: the next start reads the same config.tree.close()+ if handle is not None:+ with contextlib.suppress(OSError):+ handle.close()+++async def _migrate(launcher: list[str], env: dict[str, str], home: Path) -> bool:+ """The runtime's own migration of the agent's data; True when it succeeded."""+ started = time.monotonic()+ code = await _run_doctor(launcher, env, home, timeout_s=_MIGRATE_TIMEOUT_S)+ elapsed = time.monotonic() - started+ if code == 0:+ log.info("agent runtimes: openclaw doctor --fix for %s done in %.1f s", home.name, elapsed)+ return True+ log.warning(+ "agent runtimes: openclaw doctor --fix for %s ended with %s after %.1f s:%s",+ home.name, code, elapsed, _log_tail(home / "doctor.log", 4),+ )+ return False+++def _has_agent_data(home: Path) -> bool:+ """Whether the agent's state folder holds an OpenClaw agent database."""+ agents = home / "state" / "agents"+ try:+ return any(agents.glob("*/agent/*.sqlite"))+ except OSError: # an unreadable folder: nothing to migrate from here+ return False+++def _upgraded_since_last_start(home: Path, build: str) -> bool:+ """Whether the folder's data was last used by another OpenClaw release.++ Only a folder that recorded its release and holds a database counts: a+ new agent has nothing to migrate, and a folder from before this record+ existed is caught by the admission check when its Gateway starts.+ """+ if not build:+ return False+ try:+ previous = (home / _BUILD_MARKER).read_text(encoding="utf-8").strip()+ except OSError: # never recorded: nothing to compare with+ return False+ return bool(previous) and previous != build and _has_agent_data(home)+++def _remember_build(home: Path, build: str) -> None:+ if build:+ write_if_changed(home / _BUILD_MARKER, build + "\n")+++def admission_state(text: str) -> str:+ """``ready`` / ``pending`` / ``degraded`` from the Gateway's log of this run."""+ if _ADMISSION_DEGRADED.search(text):+ return "degraded"+ if any(marker in text for marker in _ADMISSION_PENDING) and not any(+ marker in text for marker in _ADMISSION_RECOVERED+ ):+ return "pending"+ return "ready"+++def _read_log(path: Path) -> str:+ try:+ return path.read_text(encoding="utf-8", errors="replace")+ except OSError: # not written yet: the caller polls again+ return ""+++async def _await_admission(log_path: Path, proc: asyncio.subprocess.Process) -> str:+ """Wait until the Gateway's agent database admits messages (or never will).++ Returns ``ready``, ``degraded``, or ``pending`` when :data:`_ADMISSION_WAIT_S`+ passed first (the turn then retries a refused message on its own).+ """+ deadline = time.monotonic() + _ADMISSION_WAIT_S+ started = time.monotonic()+ waited = False+ while True:+ state = admission_state(await asyncio.to_thread(_read_log, log_path))+ if state != "pending" or proc.returncode is not None:+ if waited:+ log.info("agent runtimes: OpenClaw admission %s after %.1f s",+ state, time.monotonic() - started)+ return state+ if time.monotonic() >= deadline:+ log.warning("agent runtimes: OpenClaw admission still pending after %.0f s",+ _ADMISSION_WAIT_S)+ return state+ waited = True+ await asyncio.sleep(0.25)+++def _degraded_reason(log_path: Path) -> str:+ text = _read_log(log_path)+ for line in text.splitlines():+ if _ADMISSION_DEGRADED.search(line):+ match = _DEGRADED_REASON.search(line)+ reason = match.group(1) if match else line.strip()+ reason = _SECRET_SHAPES.sub("[redacted]", " ".join(reason.split()))+ return reason[:300]+ return ""+++def _repair_failed_message(reason: str, migrated: bool) -> str:+ detail = f" ({reason})" if reason else ""+ ran = "ran" if migrated else "tried"+ return (+ "OpenClaw cannot open this agent's saved data after its update"+ f"{detail}. Dodaios {ran} OpenClaw's own repair (openclaw doctor --fix), but the "+ "data still needs it. Your message was not sent; the details are in doctor.log in "+ "the agent's OpenClaw folder."+ )def _log_tail(path: Path, lines: int = 6) -> str:diff --git a/tests/fakes/fake_acp_agent.py b/tests/fakes/fake_acp_agent.pyindex e362378427..f8fb233288 100644--- a/tests/fakes/fake_acp_agent.py+++ b/tests/fakes/fake_acp_agent.py@@ -17,6 +17,12 @@ What a prompt does depends on its text:* contains ``ASK`` — a ``session/request_permission`` first; the answer'soption id is echoed in the reply;* contains ``FAIL`` — the prompt request answers a JSON-RPC error;+* contains ``NOTREADY`` — refuses the prompt as OpenClaw does while its+ Gateway finishes its start (UNAVAILABLE "has not completed startup+ inspection") ``FAKE_ACP_REFUSALS`` times (default 1, counted in+ ``FAKE_ACP_REFUSALS_FILE``), then answers like any prompt;+* contains ``STALEDB`` — always refuses it as a database that needs+ ``openclaw doctor --fix`` (counted the same way);* contains ``HANG`` — answers, then keeps running after stdin closes (aruntime whose child holds the pipe);* contains ``SLOW`` — streams "working", then waits; a ``session/cancel``@@ -148,6 +154,19 @@ def _prompt(rid: Any, session_id: str, text: str, store: dict[str, list[str]]) -# A runtime that swallowed a failed model call and ended with no answer._send({"jsonrpc": "2.0", "id": rid, "result": {"stopReason": "end_turn"}})return+ if "NOTREADY" in text or "STALEDB" in text:+ counter = Path(os.environ.get("FAKE_ACP_REFUSALS_FILE", "refusals.txt"))+ refused = int(counter.read_text(encoding="utf-8")) if counter.is_file() else 0+ if "STALEDB" in text or refused < int(os.environ.get("FAKE_ACP_REFUSALS", "1")):+ counter.write_text(str(refused + 1), encoding="utf-8")+ message = (+ "UNAVAILABLE: Agent main has not completed startup inspection and preparation"+ if "NOTREADY" in text+ else "UNAVAILABLE: OpenClaw agent database openclaw-agent.sqlite uses schema "+ "version 24; stop active agents and run openclaw doctor --fix"+ )+ _send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32000, "message": message}})+ returnif "FAIL" in text:_send({"jsonrpc": "2.0", "id": rid, "error": {"code": -32000, "message": "model down"}})returndiff --git a/tests/fakes/fake_openclaw.py b/tests/fakes/fake_openclaw.pyindex f4ce829bbd..25479623ca 100644--- a/tests/fakes/fake_openclaw.py+++ b/tests/fakes/fake_openclaw.py@@ -10,13 +10,23 @@ Run as ``python tests/fakes/fake_openclaw.py <args>``. Understands:``FAKE_OPENCLAW_MODE``:* ``config-once`` — the Gateway exits 78 (``EX_CONFIG``) until doctor ran;-* ``doctor-hang`` — doctor writes its pid to ``doctor.pid`` and sleeps.+* ``doctor-hang`` — doctor writes its pid to ``doctor.pid`` and sleeps;+* ``schema-stale`` — the Gateway serves ``/healthz`` but logs that its agent+ database "uses schema version 24" (2026.10.8 after an update) until doctor+ ran; doctor counts its runs in ``doctor.runs``;+* ``schema-broken`` — the same, and doctor fails (exit 1) without fixing it;+* ``admission-slow`` — the Gateway logs "has not completed startup inspection+ and preparation" and, ``FAKE_OPENCLAW_ADMISSION_S`` seconds after+ ``/healthz`` answers, that the database recovered.++Every run but ``--version`` appends its argv to ``calls.log`` in the working folder."""from __future__ import annotationsimport osimport sys+import threadingimport timefrom http.server import BaseHTTPRequestHandler, HTTPServerfrom pathlib import Path@@ -33,15 +43,35 @@ class _Health(BaseHTTPRequestHandler):return+_DB = r"state\agents\main\agent\openclaw-agent.sqlite"+_STALE = (+ f"OpenClaw agent database {_DB} uses schema version 24; stop active agents and run "+ "openclaw doctor --fix to migrate session identities before using it."+)+++def _say(line: str) -> None:+ print(line, flush=True)++def main(argv: list[str]) -> int:mode = os.environ.get("FAKE_OPENCLAW_MODE", "")if argv[:1] == ["--version"]:print("OpenClaw 2026.9.8 (fake)")return 0+ with Path("calls.log").open("a", encoding="utf-8") as calls:+ calls.write(" ".join(argv) + "\n")if argv[:1] == ["doctor"]:if mode == "doctor-hang":Path("doctor.pid").write_text(str(os.getpid()), encoding="utf-8")time.sleep(600)+ runs = Path("doctor.runs")+ count = int(runs.read_text(encoding="utf-8")) if runs.exists() else 0+ runs.write_text(str(count + 1), encoding="utf-8")+ if mode == "schema-broken":+ _say("Doctor could not migrate the agent database: disk is read-only")+ return 1+ _say("Doctor migrated the agent database to schema version 25")Path("doctor.done").write_text("1", encoding="utf-8")return 0if argv[:2] == ["gateway", "run"]:@@ -52,6 +82,29 @@ def main(argv: list[str]) -> int:server = HTTPServer(("127.0.0.1", port), _Health)except OSError:return 1 # EADDRINUSE: the port was taken+ if mode in {"schema-stale", "schema-broken", "admission-slow"}:+ _say(+ "[state/agent-admission] Agent main has not completed startup inspection and "+ "preparation. Inspection and writable admission continue after the Gateway "+ "listener binds."+ )+ if mode in {"schema-stale", "schema-broken"} and not Path("doctor.done").exists():+ _say(+ "2026-10-27T16:22:41.343+01:00 [state/agent-admission] agent database remains "+ f'degraded agentId=main reason="{_STALE}" repairHint="Sessions remain '+ 'unavailable."'+ )+ elif mode in {"schema-stale", "admission-slow"}:+ delay = float(os.environ.get("FAKE_OPENCLAW_ADMISSION_S", "0"))++ def recover() -> None:+ time.sleep(delay)+ _say(+ "2026-10-27T16:54:29.657+01:00 [state/agent-admission] agent database "+ "recovered after background inspection and preparation"+ )++ threading.Thread(target=recover, daemon=True).start()server.serve_forever()return 0print(f"fake openclaw: unsupported {argv}", file=sys.stderr)diff --git a/tests/unit/agent_runtimes/test_hermes_profiles.py b/tests/unit/agent_runtimes/test_hermes_profiles.pyindex 194c5ec08d..a906b6bd0d 100644--- a/tests/unit/agent_runtimes/test_hermes_profiles.py+++ b/tests/unit/agent_runtimes/test_hermes_profiles.py@@ -140,6 +140,9 @@ def test_a_stale_agent_launcher_on_path_never_answers(tmp_path, monkeypatch):import dodaios.core.path_augment as path_augmentmonkeypatch.setattr(path_augment, "ensure_cli_paths", lambda: [])+ # This machine's own Hermes and temporary folders stay out of the picture.+ monkeypatch.setattr(hermes, "_install_locations", lambda: [])+ monkeypatch.setattr(hermes, "_in_temp", lambda _directory: False)found = hermes._binary()assert found is not None and Path(found).parent == realdiff --git a/tests/unit/agent_runtimes/test_openclaw_gateway.py b/tests/unit/agent_runtimes/test_openclaw_gateway.pyindex e7d888984b..9c5f01057d 100644--- a/tests/unit/agent_runtimes/test_openclaw_gateway.py+++ b/tests/unit/agent_runtimes/test_openclaw_gateway.py@@ -25,6 +25,10 @@ _LAUNCHER = [sys.executable, str(_FAKE)]@pytest.fixturedef home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:monkeypatch.setattr(base, "runtimes_root", lambda: tmp_path / "rt")+ # Never this machine's own OpenClaw ``--version`` for the release record.+ monkeypatch.setattr(+ openclaw.OpenClawRuntime, "_installed_build", lambda self: "OpenClaw 2026.9.8 (fake)"+ )folder = tmp_path / "rt" / "openclaw" / "hermit"folder.mkdir(parents=True)return folderdiff --git a/tests/unit/agent_runtimes/test_runtime_upgrade_safety.py b/tests/unit/agent_runtimes/test_runtime_upgrade_safety.pynew file mode 100644index 0000000000..6d26bb67ba--- /dev/null+++ b/tests/unit/agent_runtimes/test_runtime_upgrade_safety.py@@ -0,0 +1,362 @@+"""Runtime upgrades, start-up readiness and launcher resolution stay safe.++* An OpenClaw update that leaves the agent's database on an older schema gets+ OpenClaw's own migration (``openclaw doctor --fix``) with the Gateway+ stopped, then a restart; a migration that does not help is a readable+ error, never a Gateway that refuses every message.+* A Gateway that answers ``/healthz`` but has not finished its start-up+ inspection is waited for; a message it still refuses is sent again.+* Hermes is never started from a temporary folder on PATH.++Fakes only (``tests/fakes/fake_openclaw.py``, ``fake_acp_agent.py``): no+real runtime, no model call, nothing billed.+"""++from __future__ import annotations++import asyncio+import os+import sys+import time+from pathlib import Path++import pytest++from dodaios.agent_chat import runner_cli as rc+from dodaios.agent_runtimes import base, hermes, openclaw+from dodaios.agent_runtimes.base import RuntimeTurn, RuntimeUnavailable, not_ready_kind+from dodaios.agent_runtimes.model_map import ModelRoute++from .test_acp_runner import _finished, _texts, _turn++_FAKE = Path(__file__).resolve().parents[2] / "fakes" / "fake_openclaw.py"+_LAUNCHER = [sys.executable, str(_FAKE)]+_BUILD = "OpenClaw 2026.10.8 (fake)"+++def home(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> Path:+ monkeypatch.setattr(base, "runtimes_root", lambda: tmp_path / "rt")+ monkeypatch.setattr(openclaw.OpenClawRuntime, "_installed_build", lambda self: _BUILD)+ folder = tmp_path / "rt" / "openclaw" / "hermit"+ folder.mkdir(parents=True)+ return folder+++def _turn_for(workspace: Path) -> RuntimeTurn:+ return RuntimeTurn(+ agent_id="hermit",+ agent_name="Hermit",+ session_id="society:hermit",+ workspace=workspace,+ route=ModelRoute("ollama", "qwen3", "http://127.0.0.1:11434/v1", "chat_completions", None),+ resume=None,+ auto_approve=True,+ )+++def _env(**extra: str) -> dict[str, str]:+ env = dict(os.environ)+ env.update(extra)+ return env+++async def _start(runtime: openclaw.OpenClawRuntime, home: Path, env: dict[str, str]):+ return await runtime._start_gateway(+ _turn_for(home.parent), "hermit", home, openclaw._free_port(), "tok", "h", _LAUNCHER, env+ )+++def _calls(home: Path) -> list[str]:+ path = home / "calls.log"+ return path.read_text(encoding="utf-8").splitlines() if path.is_file() else []+++def _with_agent_db(home: Path) -> None:+ db = home / "state" / "agents" / "main" / "agent" / "openclaw-agent.sqlite"+ db.parent.mkdir(parents=True)+ db.write_bytes(b"")+++# ------------------------------------------------- (a) data migration+++def test_a_stale_agent_database_is_migrated_and_the_gateway_restarted(home):+ async def run() -> None:+ runtime = openclaw.OpenClawRuntime()+ gateway = await _start(runtime, home, _env(FAKE_OPENCLAW_MODE="schema-stale"))+ try:+ assert gateway.alive() and await openclaw._healthy(gateway.port)+ assert (home / "doctor.runs").read_text(encoding="utf-8") == "1"+ # Stopped, migrated, started again: in that order.+ calls = [line.split()[0] for line in _calls(home)]+ assert calls == ["gateway", "doctor", "gateway"]+ log = (home / "gateway.log").read_text(encoding="utf-8")+ assert "degraded" not in log and "recovered" in log+ assert "migrated" in (home / "doctor.log").read_text(encoding="utf-8")+ assert (home / "runtime-build.txt").read_text(encoding="utf-8").strip() == _BUILD+ finally:+ await runtime._stop_gateway(gateway)++ asyncio.run(run())+++def test_a_migration_that_does_not_help_is_a_readable_error_without_a_gateway(home):+ async def run() -> None:+ runtime = openclaw.OpenClawRuntime()+ with pytest.raises(RuntimeUnavailable) as caught:+ await _start(runtime, home, _env(FAKE_OPENCLAW_MODE="schema-broken"))+ message = str(caught.value)+ assert "openclaw doctor --fix" in message+ assert "schema version 24" in message+ assert "not sent" in message+ # Exactly one repair attempt, and no Gateway left running.+ assert (home / "doctor.runs").read_text(encoding="utf-8") == "1"+ assert not (home / "gateway.pid").exists()+ assert runtime._gateways == {}+ assert "read-only" in (home / "doctor.log").read_text(encoding="utf-8")++ asyncio.run(run())+++def test_a_version_change_migrates_before_the_gateway_starts(home):+ _with_agent_db(home)+ (home / "runtime-build.txt").write_text("OpenClaw 2026.9.8 (fake)\n", encoding="utf-8")++ async def run() -> None:+ runtime = openclaw.OpenClawRuntime()+ gateway = await _start(runtime, home, _env())+ try:+ assert [line.split()[0] for line in _calls(home)] == ["doctor", "gateway"]+ assert (home / "runtime-build.txt").read_text(encoding="utf-8").strip() == _BUILD+ finally:+ await runtime._stop_gateway(gateway)+ # The same release again: no migration.+ gateway = await _start(runtime, home, _env())+ try:+ assert [line.split()[0] for line in _calls(home)] == ["doctor", "gateway", "gateway"]+ finally:+ await runtime._stop_gateway(gateway)++ asyncio.run(run())+++def test_a_new_agent_or_an_unrecorded_folder_is_not_migrated_up_front(home):+ _with_agent_db(home) # data, but no release recorded yet++ async def run() -> None:+ runtime = openclaw.OpenClawRuntime()+ gateway = await _start(runtime, home, _env())+ try:+ assert [line.split()[0] for line in _calls(home)] == ["gateway"]+ finally:+ await runtime._stop_gateway(gateway)++ asyncio.run(run())+ assert not openclaw._upgraded_since_last_start(home / "missing", _BUILD)+++def test_the_admission_state_reads_the_gateways_own_log():+ pending = (+ "[state/agent-admission] Agent main has not completed startup inspection and "+ "preparation. Inspection and writable admission continue after the Gateway "+ "listener binds."+ )+ recovered = "agent database recovered after background inspection and preparation"+ degraded = (+ '[state/agent-admission] agent database remains degraded agentId=main reason="OpenClaw '+ 'agent database C:\\x\\openclaw-agent.sqlite uses schema version 24; run openclaw '+ 'doctor --fix"'+ )+ assert openclaw.admission_state("") == "ready" # older releases log none of it+ assert openclaw.admission_state(pending) == "pending"+ assert openclaw.admission_state(pending + "\n" + recovered) == "ready"+ assert openclaw.admission_state(pending + "\n" + degraded) == "degraded"+ # A config hint that names doctor --fix is no database problem.+ assert openclaw.admission_state("Run openclaw doctor --fix to drop legacy keys") == "ready"+++# ------------------------------------------------- (b) start-up readiness+++def test_a_gateway_is_handed_out_only_after_its_startup_inspection(home):+ async def run() -> None:+ runtime = openclaw.OpenClawRuntime()+ started = time.monotonic()+ gateway = await _start(+ runtime,+ home,+ _env(FAKE_OPENCLAW_MODE="admission-slow", FAKE_OPENCLAW_ADMISSION_S="1.5"),+ )+ try:+ assert time.monotonic() - started >= 1.4+ assert "recovered" in (home / "gateway.log").read_text(encoding="utf-8")+ assert not (home / "doctor.runs").exists()+ finally:+ await runtime._stop_gateway(gateway)++ asyncio.run(run())+++def test_an_admission_that_never_ends_still_hands_out_the_gateway(home, monkeypatch):+ monkeypatch.setattr(openclaw, "_ADMISSION_WAIT_S", 0.5)++ async def run() -> None:+ runtime = openclaw.OpenClawRuntime()+ gateway = await _start(+ runtime,+ home,+ _env(FAKE_OPENCLAW_MODE="admission-slow", FAKE_OPENCLAW_ADMISSION_S="60"),+ )+ try:+ # The turn sends; a refusal is sent again by the runner.+ assert gateway.alive()+ finally:+ await runtime._stop_gateway(gateway)++ asyncio.run(run())+++def test_runtime_refusals_are_classified():+ assert not_ready_kind(+ "UNAVAILABLE: Agent main has not completed startup inspection and preparation"+ ) == "starting"+ assert not_ready_kind(+ "openclaw-agent.sqlite uses schema version 24; stop active agents and run "+ "openclaw doctor --fix"+ ) == "repair"+ stores = "ACP session/new failed: Session stores for agent main are unavailable"+ assert not_ready_kind(stores) == "repair"+ assert not_ready_kind("model down") == ""+ assert not_ready_kind(None) == ""+++def quick_retries(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> Path:+ monkeypatch.setattr(rc, "_RUNTIME_READY_FIRST_DELAY_S", 0.05)+ monkeypatch.setattr(rc, "_RUNTIME_READY_MAX_DELAY_S", 0.1)+ counter = tmp_path / "refusals.txt"+ monkeypatch.setenv("FAKE_ACP_REFUSALS_FILE", str(counter))+ return counter+++def test_a_message_refused_while_the_runtime_starts_is_sent_again(+ monkeypatch, tmp_path, quick_retries+):+ monkeypatch.setenv("FAKE_ACP_REFUSALS", "3")+ _, events, _, resumes = _turn(monkeypatch, tmp_path, "NOTREADY hello")+ assert _finished(events)["status"] == "done"+ assert _texts(events) == ["echo: NOTREADY hello"]+ assert quick_retries.read_text(encoding="utf-8") == "3"+ assert len(resumes) == 4 # three refusals, then the answer+++def test_a_runtime_that_stays_not_ready_ends_the_turn_after_the_bound(+ monkeypatch, tmp_path, quick_retries+):+ monkeypatch.setattr(rc, "_RUNTIME_READY_WAIT_S", 0.4)+ monkeypatch.setenv("FAKE_ACP_REFUSALS", "1000")+ _, events, _, resumes = _turn(monkeypatch, tmp_path, "NOTREADY hello")+ finished = _finished(events)+ assert finished["status"] == "error"+ assert "startup inspection" in finished["error"]+ assert 2 <= len(resumes) < 20+++def test_a_database_that_needs_repair_gets_one_restart_then_its_error(+ monkeypatch, tmp_path, quick_retries+):+ _, events, _, resumes = _turn(monkeypatch, tmp_path, "STALEDB hello")+ finished = _finished(events)+ assert finished["status"] == "error"+ assert "doctor --fix" in finished["error"]+ assert len(resumes) == 2+++def test_other_errors_are_never_sent_again(monkeypatch, tmp_path, quick_retries):+ _, events, _, resumes = _turn(monkeypatch, tmp_path, "FAIL")+ assert _finished(events)["status"] == "error"+ assert len(resumes) == 1+++# ------------------------------------------------- (c) Hermes launcher+++def _launcher_in(folder: Path) -> Path:+ folder.mkdir(parents=True, exist_ok=True)+ launcher = folder / ("hermes.cmd" if os.name == "nt" else "hermes")+ launcher.write_text("", encoding="utf-8")+ launcher.chmod(0o755)+ return launcher+++def _no_path_augment(monkeypatch: pytest.MonkeyPatch) -> None:+ import dodaios.core.path_augment as path_augment++ monkeypatch.setattr(path_augment, "ensure_cli_paths", lambda: [])+++def _temp_is(monkeypatch: pytest.MonkeyPatch, *roots: Path) -> None:+ """Only ``roots`` count as temporary (pytest's own folder may be in Temp)."""+ import re++ monkeypatch.setattr(hermes, "_temp_roots", lambda: [hermes._norm(str(r)) for r in roots])+ monkeypatch.setattr(hermes, "_WINDOWS_TEMP", re.compile(r"(?!)"))+++def test_hermes_from_a_temporary_audit_folder_never_runs(tmp_path, monkeypatch):+ temp = tmp_path / "Temp"+ audit = temp / "jarvis-hermes-audit-m1e61qu4" / "runtimes" / "hermes" / "bin"+ real = tmp_path / "Programs" / "hermes" / "bin"+ _launcher_in(audit)+ _launcher_in(real)+ monkeypatch.setattr(hermes, "_install_locations", lambda: [])+ _temp_is(monkeypatch, temp)+ monkeypatch.setenv("PATH", os.pathsep.join([str(audit), str(real)]))+ _no_path_augment(monkeypatch)+ found = hermes._binary()+ assert found is not None and Path(found).parent == real+ # Only the temporary one on PATH: Hermes counts as not installed.+ monkeypatch.setenv("PATH", str(audit))+ assert hermes._binary() is None+++ "entry",+ [+ r"C:\Users\Ada\AppData\Local\Temp\jarvis-hermes-audit-m1e61qu4\runtimes\hermes\bin",+ r"C:\Users\ADA~1\AppData\Local\Temp\jarvis-hermes-audit-x\runtimes\hermes\bin",+ ],+)+def test_a_windows_temp_folder_is_recognised_under_any_profile_spelling(entry):+ assert hermes._in_temp(entry)+ assert not hermes._in_temp(r"C:\Users\Ada\AppData\Local\hermes\bin")+++def test_the_official_install_answers_before_path(tmp_path, monkeypatch):+ managed = _launcher_in(tmp_path / "Local" / "hermes" / "bin")+ other = tmp_path / "elsewhere" / "bin"+ _launcher_in(other)+ monkeypatch.setattr(hermes, "_install_locations", lambda: [managed])+ _temp_is(monkeypatch)+ monkeypatch.setenv("PATH", str(other))+ _no_path_augment(monkeypatch)+ assert hermes._binary() == str(managed)+ # Without the official install, PATH answers.+ monkeypatch.setattr(hermes, "_install_locations", lambda: [tmp_path / "missing" / "hermes"])+ found = hermes._binary()+ assert found is not None and Path(found).parent == other+++def test_the_windows_install_location_keeps_its_cmd_launcher(tmp_path, monkeypatch):+ monkeypatch.setattr(hermes, "is_windows", lambda: True)+ monkeypatch.setenv("LOCALAPPDATA", str(tmp_path))+ bin_dir = tmp_path / "hermes" / "bin"+ assert hermes._install_locations() == [bin_dir / "hermes.exe", bin_dir / "hermes.cmd"]+++def test_the_user_path_is_never_edited_for_a_temporary_entry(tmp_path, monkeypatch):+ from dodaios.agent_runtimes import path_cleanup++ entry = r"C:\Users\Ada\AppData\Local\Temp\jarvis-hermes-audit-m1e61qu4\runtimes\hermes\bin"+ assert not path_cleanup.is_jarvis_hermes_bin(entry)