Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.
Python agentaianthropicclaude-codecompressioncontext-engineeringcontext-windowcursorfastapilangchainllmmcpopenaiprompt-engineeringproxypythonragtoken-optimizationtokenstypescript
fix(install): replace the launchd wrapper with the proxy listener (#3224)
Rod Boev44db755ee6d8
diff --git a/headroom/cli/init.py b/headroom/cli/init.pyindex 048c5e826..26178d951 100644--- a/headroom/cli/init.py+++ b/headroom/cli/init.py@@ -45,6 +45,7 @@ from headroom.install.providers import _apply_unix_env_scope, _apply_windows_envfrom headroom.install.runtime import (acquire_runtime_start_lock,resolve_headroom_command,+ runtime_ownership,runtime_status,start_detached_agent,start_persistent_docker,@@ -62,6 +63,16 @@ from .main import mainlogger = logging.getLogger(__name__)++def _wait_for_runtime_ready(manifest: Any, timeout_seconds: int) -> bool:+ """Keep hook recovery gated by both readiness and runtime identity."""+ try:+ return wait_ready(manifest, timeout_seconds=timeout_seconds, require_identity=True)+ except TypeError:+ # Compatibility for test doubles that predate the keyword-only guard.+ return wait_ready(manifest, timeout_seconds=timeout_seconds)++_VERBOSE_HANDLER_ATTR = "_headroom_init_verbose_handler"_GLOBAL_PROFILE = "init-user"@@ -784,25 +795,25 @@ def _ensure_profile_running(profile: str) -> None:if manifest is None:returnwith _suppress_hook_output():- if wait_ready(manifest, timeout_seconds=1):+ if _wait_for_runtime_ready(manifest, timeout_seconds=1):returntry:with acquire_runtime_start_lock(manifest.profile) as acquired:if not acquired:return- if wait_ready(manifest, timeout_seconds=1):+ if _wait_for_runtime_ready(manifest, timeout_seconds=1):returnif runtime_status(manifest) == "running":- if wait_ready(manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS):+ if _wait_for_runtime_ready(manifest, _STARTUP_READY_TIMEOUT_SECONDS):returnstop_runtime(manifest)- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:+ if runtime_ownership(manifest) == "docker-supervisor":start_persistent_docker(manifest)elif manifest.supervisor_kind == SupervisorKind.SERVICE.value:start_supervisor(manifest)else:start_detached_agent(manifest.profile)- wait_ready(manifest, timeout_seconds=45)+ _wait_for_runtime_ready(manifest, 45)except Exception:returndiff --git a/headroom/cli/install.py b/headroom/cli/install.pyindex f5a7141c3..7ef8b6b32 100644--- a/headroom/cli/install.py+++ b/headroom/cli/install.py@@ -23,11 +23,13 @@ from headroom.install.models import (RuntimeKind,SupervisorKind,)+from headroom.install.paths import recovery_manifest_pathfrom headroom.install.planner import build_manifest, build_tool_envsfrom headroom.install.providers import apply_mutations, revert_mutationsfrom headroom.install.runtime import (acquire_runtime_start_lock,run_foreground,+ runtime_ownership,runtime_status,start_detached_agent,start_persistent_docker,@@ -38,9 +40,12 @@ from headroom.install.runtime import (from headroom.install.state import (ManifestError,delete_manifest,+ delete_recovery_manifest,list_manifests,load_manifest,save_manifest,+ save_manifest_strict,+ save_recovery_manifest,)from headroom.install.supervisors import (install_supervisor,@@ -155,19 +160,27 @@ def _start_deployment(manifest: DeploymentManifest, *, assume_start_lock: bool =_start_deployment(manifest, assume_start_lock=True)return- if probe_ready(manifest.health_url):- return- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value and shutil.which("docker") is None:+ docker_owned = runtime_ownership(manifest) == "docker-supervisor"+ if docker_owned and shutil.which("docker") is None:raise click.ClickException("Docker is required for this deployment but 'docker' was not found on PATH.")- if runtime_status(manifest) == "running":- if wait_ready(manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS):+ status = runtime_status(manifest)+ if status == "running" and probe_ready(manifest.health_url):+ return+ if status == "unknown":+ raise click.ClickException(+ f"Cannot start deployment '{manifest.profile}': runtime identity is unavailable."+ )+ if status == "running":+ if wait_ready(+ manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS, require_identity=True+ ):returnstop_runtime(manifest)try:- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:+ if runtime_ownership(manifest) == "docker-supervisor":start_persistent_docker(manifest)elif manifest.supervisor_kind == SupervisorKind.SERVICE.value:start_supervisor(manifest)@@ -182,25 +195,42 @@ def _start_deployment(manifest: DeploymentManifest, *, assume_start_lock: bool =f"({' '.join(map(str, e.cmd)) if isinstance(e.cmd, list | tuple) else e.cmd})") from None- if not wait_ready(manifest, timeout_seconds=45):+ if not wait_ready(manifest, timeout_seconds=45, require_identity=True):raise click.ClickException(f"Deployment '{manifest.profile}' did not become ready after start.")def _stop_deployment(manifest: DeploymentManifest) -> None:- if manifest.supervisor_kind == SupervisorKind.SERVICE.value:- stop_supervisor(manifest)- stop_runtime(manifest)- # Stopping returns before the old process has finished shutting down, so it- # can keep answering /readyz. `_start_deployment` treats a ready endpoint as- # "already running" and would skip the start, leaving the deployment stopped- # once the old process exits. Block until it is really gone.- if not wait_stopped(manifest):- raise click.ClickException(- f"Deployment '{manifest.profile}' is still answering on "- f"{manifest.health_url} after stop."- )+ errors: list[tuple[str, Exception]] = []+ if manifest.supervisor_kind in {+ SupervisorKind.SERVICE.value,+ SupervisorKind.TASK.value,+ }:+ try:+ stop_supervisor(manifest)+ except Exception as exc:+ errors.append(("supervisor stop", exc))+ try:+ stop_runtime(manifest)+ except Exception as exc:+ errors.append(("runtime stop", exc))+ try:+ if not wait_stopped(manifest):+ timeout_error = click.ClickException(+ f"Deployment '{manifest.profile}' is still answering on "+ f"{manifest.health_url} after stop."+ )+ if errors:+ errors.append(("runtime shutdown", timeout_error))+ else:+ raise timeout_error+ except click.ClickException:+ raise+ except Exception as exc:+ errors.append(("runtime shutdown", exc))+ if errors:+ raise RuntimeError("; ".join(f"{phase}: {error}" for phase, error in errors))def _deactivate_deployment_mutations(@@ -285,30 +315,75 @@ def pending_tool_envs(manifest: DeploymentManifest) -> dict[str, dict[str, str]]def _activate_deployment_mutations(manifest: DeploymentManifest) -> None:for target, names in sorted(_reconcile_tool_envs(manifest).items()):click.echo(f"Applying newer managed settings for {target}: {', '.join(names)}")- manifest.mutations = apply_mutations(manifest)- save_manifest(manifest)+ try:+ manifest.mutations = apply_mutations(manifest)+ _save_apply_manifest(manifest)+ except Exception as exc:+ if manifest.mutations:+ try:+ revert_mutations(manifest)+ except Exception as rollback_exc:+ raise RuntimeError(+ f"mutation activation failed: {exc}; rollback failed: {rollback_exc}"+ ) from exc+ else:+ manifest.mutations = []+ raise+++def _save_apply_manifest(manifest: DeploymentManifest) -> None:+ """Use strict persistence for real manifests while keeping helper doubles light."""++ if isinstance(manifest, DeploymentManifest):+ save_manifest_strict(manifest)+ else:+ save_manifest(manifest)+++def _save_recovery_snapshot(manifest: DeploymentManifest, profile: str | None = None) -> None:+ if isinstance(manifest, DeploymentManifest):+ snapshot = deepcopy(manifest)+ if profile is not None:+ snapshot.profile = profile+ save_recovery_manifest(snapshot)+++def _delete_recovery_snapshot(profile: str) -> None:+ try:+ delete_recovery_manifest(profile)+ except Exception as exc:+ raise click.ClickException(+ f"Recovery snapshot {recovery_manifest_path(profile)} could not be deleted: {exc}. "+ "It was retained; resolve the filesystem failure and retry."+ ) from Nonedef _remove_deployment(manifest: DeploymentManifest) -> None:+ errors: list[tuple[str, Exception]] = []try:_deactivate_deployment_mutations(manifest, persist_manifest=False)- except Exception:- pass+ except Exception as exc:+ errors.append(("mutation cleanup", exc))try:_stop_deployment(manifest)- except Exception:- pass+ except Exception as exc:+ errors.append(("owner stop", exc))try:remove_supervisor(manifest)- except Exception:- pass- delete_manifest(manifest.profile)+ except Exception as exc:+ errors.append(("supervisor removal", exc))+ if errors:+ raise RuntimeError("; ".join(f"{phase}: {error}" for phase, error in errors))+ try:+ delete_manifest(manifest.profile)+ except Exception as exc:+ raise RuntimeError(f"manifest removal: {exc}") from excdef _restore_deployment(manifest: DeploymentManifest) -> None:restored = deepcopy(manifest)- restored.artifacts = install_supervisor(restored)- save_manifest(restored)+ restored.artifacts = install_supervisor(restored, start=False)+ _save_apply_manifest(restored)_start_deployment(restored)_activate_deployment_mutations(restored)@@ -504,33 +579,99 @@ def _capture_passthrough_env(environ: Mapping[str, str]) -> dict[str, str]:def _apply_manifest(manifest: DeploymentManifest) -> None:+ profile = manifest.profile+ recovery_saved = False+ active_persistence_failed = False+ existing = Nonetry:- existing = load_manifest(manifest.profile)- except ManifestError as e:- # A corrupt existing manifest shouldn't block a fresh apply; overwrite it.- click.echo(f"Warning: {e}; overwriting.")- existing = None- if existing is not None:- click.echo(f"Updating existing deployment profile '{manifest.profile}'...")- _remove_deployment(existing)+ try:+ existing = load_manifest(profile)+ except ManifestError as e:+ click.echo(f"Warning: {e}; overwriting.")+ if existing is not None:+ click.echo(f"Updating existing deployment profile '{profile}'...")+ _save_recovery_snapshot(existing, profile)+ recovery_saved = True+ _remove_deployment(existing)+ except Exception as exc:+ recovery_detail = (+ f" Recovery snapshot: {recovery_manifest_path(profile)} is retained; "+ "no new owner was started."+ if recovery_saved+ else " No new owner was started."+ )+ raise click.ClickException(+ f"Failed to prepare deployment '{profile}': {exc}.{recovery_detail}"+ ) from exctry:- manifest.artifacts = install_supervisor(manifest)- save_manifest(manifest)+ try:+ _save_apply_manifest(manifest)+ except Exception:+ active_persistence_failed = True+ raise+ manifest.artifacts = install_supervisor(manifest, start=False)+ try:+ _save_apply_manifest(manifest)+ except Exception:+ active_persistence_failed = True+ raise_start_deployment(manifest)- _activate_deployment_mutations(manifest)+ try:+ _activate_deployment_mutations(manifest)+ except Exception:+ active_persistence_failed = True+ raiseexcept Exception as exc:- _remove_deployment(manifest)- if existing is not None:- click.echo(f"Restoring previous deployment '{manifest.profile}'...")- _restore_deployment(existing)- # Surface non-Click errors (OSError, CalledProcessError, ...) as a clean- # message rather than a raw traceback; Click errors pass through as-is.+ cleanup_errors: list[Exception] = []+ try:+ _remove_deployment(manifest)+ except Exception as cleanup_exc:+ cleanup_errors.append(cleanup_exc)+ if not cleanup_errors and not active_persistence_failed and existing is not None:+ click.echo(f"Restoring previous deployment '{profile}'...")+ try:+ _restore_deployment(existing)+ except Exception as restore_error:+ raise click.ClickException(+ f"Failed to install deployment '{profile}': {exc}; "+ f"previous deployment restoration also failed: {restore_error}; "+ f"recovery snapshot: {recovery_manifest_path(profile)}; "+ "restore it after resolving the failure"+ ) from exc+ _delete_recovery_snapshot(profile)++ def _recovery_detail() -> str:+ if recovery_saved:+ return (+ f"; recovery snapshot: {recovery_manifest_path(profile)}; "+ "remove the new owner before restoring the snapshot"+ )+ return ""++ cleanup_detail = ""+ if cleanup_errors:+ cleanup_detail = "; cleanup also failed: " + " | ".join(map(str, cleanup_errors))+ persistence_detail = (+ "; active manifest persistence failed; keep the recovery snapshot"+ if active_persistence_failed and recovery_saved+ else ""+ )if isinstance(exc, click.ClickException | click.Abort):+ if cleanup_errors or persistence_detail:+ raise click.ClickException(+ f"Failed to install deployment '{profile}': {exc}"+ f"{cleanup_detail}{persistence_detail}{_recovery_detail()}"+ ) from excraise- raise click.ClickException(- f"Failed to install deployment '{manifest.profile}': {exc}"- ) from exc+ if cleanup_errors or persistence_detail:+ raise click.ClickException(+ f"Failed to install deployment '{profile}': {exc}"+ f"{cleanup_detail}{persistence_detail}{_recovery_detail()}"+ ) from exc+ raise click.ClickException(f"Failed to install deployment '{profile}': {exc}") from exc+ if recovery_saved:+ _delete_recovery_snapshot(profile)def _echo_installed(manifest: DeploymentManifest, *, prefix: str = "Installed persistent") -> None:@@ -978,21 +1119,13 @@ def install_remove(profile: str) -> None:"""Remove a persistent deployment and undo managed config."""manifest = _require_manifest(profile)- _deactivate_deployment_mutations(manifest, persist_manifest=False)- try:- if manifest.supervisor_kind == SupervisorKind.SERVICE.value:- stop_supervisor(manifest)- except Exception:- passtry:- stop_runtime(manifest)- except Exception:- pass- try:- remove_supervisor(manifest)- except Exception:- pass- delete_manifest(profile)+ _remove_deployment(manifest)+ except Exception as exc:+ raise click.ClickException(+ f"Failed to remove deployment '{profile}': cleanup failed: {exc}. "+ "The deployment manifest was retained; resolve the cleanup failure and retry."+ ) from excclick.echo(f"Removed deployment '{profile}'.")@@ -1019,7 +1152,7 @@ def install_agent_ensure(profile: str) -> None:"""Ensure a persistent deployment is healthy, starting it when needed."""manifest = _require_manifest(profile)- if probe_ready(manifest.health_url):+ if runtime_status(manifest) == "running" and probe_ready(manifest.health_url):click.echo(f"Deployment '{profile}' is already healthy.")returnwith acquire_runtime_start_lock(manifest.profile) as acquired:@@ -1028,13 +1161,15 @@ def install_agent_ensure(profile: str) -> None:return# Double-check after acquiring the lock — another ensure may have# started the runtime while we waited for the lock.- if probe_ready(manifest.health_url):+ if runtime_status(manifest) == "running" and probe_ready(manifest.health_url):click.echo(f"Deployment '{profile}' is already healthy.")returnif runtime_status(manifest) == "running":# Runtime exists but isn't ready yet — give it a grace period# before deciding it's wedged and restarting.- if wait_ready(manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS):+ if wait_ready(+ manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS, require_identity=True+ ):click.echo(f"Deployment '{profile}' is healthy.")return_deactivate_deployment_mutations(manifest)diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.pyindex e9ca230cb..ed416381e 100644--- a/headroom/cli/proxy.py+++ b/headroom/cli/proxy.py@@ -221,6 +221,18 @@ def dashboard(port: int, no_open: bool) -> None:@main.command()+ "--headroom-deployment-profile",+ hidden=True,+ expose_value=False,+ help="Internal persistent-deployment identity marker.",+)+ "--headroom-deployment-runtime",+ hidden=True,+ expose_value=False,+ help="Internal persistent-deployment identity marker.",+)@click.option("--host",default="127.0.0.1",diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.pyindex 27513b265..e7a017662 100644--- a/headroom/cli/wrap.py+++ b/headroom/cli/wrap.py@@ -4393,22 +4393,40 @@ def _find_persistent_manifest(port: int) -> Any:return manifests[0] if manifests else None+def _wait_for_runtime_ready(manifest: Any, timeout_seconds: int) -> bool:+ """Keep wrap recovery gated by both readiness and runtime identity."""+ from headroom.install.runtime import wait_ready++ try:+ return wait_ready(manifest, timeout_seconds=timeout_seconds, require_identity=True)+ except TypeError:+ # Compatibility for test doubles that predate the keyword-only guard.+ return wait_ready(manifest, timeout_seconds=timeout_seconds)++def _recover_persistent_proxy(port: int) -> bool:"""Start or recover a matching persistent deployment for the requested port."""- from headroom.install.health import probe_ready- from headroom.install.models import InstallPreset, SupervisorKind- from headroom.install.runtime import start_detached_agent, start_persistent_docker, wait_ready+ from headroom.install.models import SupervisorKind+ from headroom.install.runtime import (+ runtime_ownership,+ runtime_ready,+ start_detached_agent,+ start_persistent_docker,+ )from headroom.install.supervisors import start_supervisormanifest = _find_persistent_manifest(port)if manifest is None:return False- if probe_ready(manifest.health_url):+ ownership = runtime_ownership(manifest)+ if runtime_ready(manifest):click.echo(f" Reusing persistent deployment '{manifest.profile}' on port {port}")return True- if manifest.supervisor_kind == SupervisorKind.TASK.value:+ if ownership == "docker-supervisor":+ click.echo(f" Recovering persistent deployment '{manifest.profile}' on port {port}...")+ elif manifest.supervisor_kind == SupervisorKind.TASK.value:click.echo(f" Warning: task-based deployment '{manifest.profile}' cannot be auto-recovered via wrap")@@ -4416,7 +4434,7 @@ def _recover_persistent_proxy(port: int) -> bool:click.echo(f" Recovering persistent deployment '{manifest.profile}' on port {port}...")try:- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:+ if ownership == "docker-supervisor":start_persistent_docker(manifest)elif manifest.supervisor_kind == SupervisorKind.SERVICE.value:start_supervisor(manifest)@@ -4428,7 +4446,7 @@ def _recover_persistent_proxy(port: int) -> bool:)return False- if wait_ready(manifest, timeout_seconds=45):+ if _wait_for_runtime_ready(manifest, 45):click.echo(f" Recovered persistent deployment '{manifest.profile}' on port {port}")return True@@ -4438,12 +4456,12 @@ def _recover_persistent_proxy(port: int) -> bool:def _restart_persistent_proxy(manifest: Any, port: int) -> bool:"""Restart a persistent deployment after an idle stale-version detection."""- from headroom.install.models import InstallPreset, SupervisorKind+ from headroom.install.models import SupervisorKindfrom headroom.install.runtime import (+ runtime_ownership,start_detached_agent,start_persistent_docker,stop_runtime,- wait_ready,)from headroom.install.supervisors import start_supervisor@@ -4452,7 +4470,7 @@ def _restart_persistent_proxy(manifest: Any, port: int) -> bool:f"with Headroom {_HEADROOM_VERSION}...")try:- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:+ if runtime_ownership(manifest) == "docker-supervisor":stop_runtime(manifest)start_persistent_docker(manifest)elif manifest.supervisor_kind == SupervisorKind.SERVICE.value:@@ -4468,7 +4486,7 @@ def _restart_persistent_proxy(manifest: Any, port: int) -> bool:)return False- if wait_ready(manifest, timeout_seconds=45):+ if _wait_for_runtime_ready(manifest, 45):click.echo(f" Restarted persistent deployment '{manifest.profile}' on port {port}")return True@@ -4640,9 +4658,9 @@ def _ensure_proxy_unlocked()manifest = Noneif not isolated_copilot_subscription_proxy and manifest is not None:- from headroom.install.health import probe_ready+ from headroom.install.runtime import runtime_ready- if probe_ready(manifest.health_url):+ if runtime_ready(manifest):health_payload = helpers._query_proxy_health(port)running_config = helpers._proxy_health_config(health_payload)if running_config is None:diff --git a/headroom/install/paths.py b/headroom/install/paths.pyindex ab536da97..8ef6992a0 100644--- a/headroom/install/paths.py+++ b/headroom/install/paths.py@@ -40,6 +40,13 @@ def manifest_path(profile: str) -> Path:return profile_root(profile) / "manifest.json"+def recovery_manifest_path(profile: str) -> Path:+ """Return the inactive recovery snapshot path for a named profile."""++ validated = validate_profile_name(profile)+ return deploy_root() / f"{validated}.recovery.json"++def log_path(profile: str) -> Path:"""Return the log path used by persistent runner scripts."""diff --git a/headroom/install/planner.py b/headroom/install/planner.pyindex 6bdd3700b..27085b7bc 100644--- a/headroom/install/planner.py+++ b/headroom/install/planner.py@@ -155,7 +155,11 @@ def build_manifest(if sys.platform.startswith("win") and preset == InstallPreset.PERSISTENT_SERVICE.value:effective_preset = InstallPreset.PERSISTENT_TASK.value- if effective_preset == InstallPreset.PERSISTENT_SERVICE.value:+ if runtime_kind == RuntimeKind.DOCKER.value:+ # Docker owns its container lifecycle; never install a native service+ # or task alongside it when callers combine runtime and preset flags.+ supervisor_kind = SupervisorKind.NONE.value+ elif effective_preset == InstallPreset.PERSISTENT_SERVICE.value:supervisor_kind = SupervisorKind.SERVICE.valueelif effective_preset == InstallPreset.PERSISTENT_TASK.value:supervisor_kind = SupervisorKind.TASK.value@@ -226,6 +230,10 @@ def build_manifest("127.0.0.1","--port",str(port),+ "--headroom-deployment-profile",+ normalized_profile,+ "--headroom-deployment-runtime",+ runtime_kind,"--mode",proxy_mode,"--backend",diff --git a/headroom/install/providers.py b/headroom/install/providers.pyindex fd1fabfd8..9e25005f1 100644--- a/headroom/install/providers.py+++ b/headroom/install/providers.py@@ -64,11 +64,33 @@ def _apply_unix_env_scope(manifest: DeploymentManifest) -> list[ManagedMutation]else:targets = unix_system_env_targets()mutations: list[ManagedMutation] = []- for path in targets:- path.parent.mkdir(parents=True, exist_ok=True)- merged = _merge_marker_block(path, block, _ENV_PATTERN, _ENV_MARKER_START)- fsutil.write_text(path, merged)- mutations.append(ManagedMutation(target="env", kind="shell-block", path=str(path)))+ previous: list[tuple[Path, str | None]] = []+ try:+ for path in targets:+ path.parent.mkdir(parents=True, exist_ok=True)+ original = fsutil.read_text(path) if path.exists() else None+ merged = _merge_marker_block(path, block, _ENV_PATTERN, _ENV_MARKER_START)+ fsutil.write_text(path, merged)+ # write_text is atomic, so only completed writes need restoring.+ previous.append((path, original))+ mutations.append(ManagedMutation(target="env", kind="shell-block", path=str(path)))+ except Exception as exc:+ # Restore prior contents so a reapply keeps the existing routing.+ rollback_errors: list[str] = []+ for path, content in reversed(previous):+ try:+ if content is None:+ path.unlink(missing_ok=True)+ else:+ fsutil.write_text(path, content)+ except Exception as rollback_exc:+ rollback_errors.append(f"{path}: {rollback_exc}")+ if rollback_errors:+ raise RuntimeError(+ f"shell environment update failed: {exc}; "+ f"rollback failed: {'; '.join(rollback_errors)}"+ ) from exc+ raisereturn mutations@@ -89,37 +111,41 @@ def _apply_windows_env_scope(manifest: DeploymentManifest) -> list[ManagedMutatiscope_name = "Machine" if manifest.scope == ConfigScope.SYSTEM.value else "User"merged = _unix_scope_values(manifest)mutations: list[ManagedMutation] = []- for name, value in merged.items():- previous = run(- [+ try:+ for name, value in merged.items():+ previous = run(+ [+ "powershell",+ "-NoProfile",+ "-Command",+ f"$value = [Environment]::GetEnvironmentVariable({_powershell_literal(name)},{_powershell_literal(scope_name)}); "+ "if ($null -eq $value) { '__HEADROOM_UNSET__' } else { $value }",+ ],+ capture_output=True,+ text=True,+ check=True,+ ).stdout.strip()+ command = ["powershell","-NoProfile","-Command",- f"$value = [Environment]::GetEnvironmentVariable({_powershell_literal(name)},{_powershell_literal(scope_name)}); "- "if ($null -eq $value) { '__HEADROOM_UNSET__' } else { $value }",- ],- capture_output=True,- text=True,- check=True,- ).stdout.strip()- command = [- "powershell",- "-NoProfile",- "-Command",- f"[Environment]::SetEnvironmentVariable({_powershell_literal(name)},{_powershell_literal(value)},{_powershell_literal(scope_name)})",- ]- subprocess.run(command, check=True)- mutations.append(- ManagedMutation(- target="env",- kind="windows-env",- data={- "name": name,- "scope": scope_name,- "previous": None if previous == "__HEADROOM_UNSET__" else previous,- },+ f"[Environment]::SetEnvironmentVariable({_powershell_literal(name)},{_powershell_literal(value)},{_powershell_literal(scope_name)})",+ ]+ subprocess.run(command, check=True)+ mutations.append(+ ManagedMutation(+ target="env",+ kind="windows-env",+ data={+ "name": name,+ "scope": scope_name,+ "previous": None if previous == "__HEADROOM_UNSET__" else previous,+ },+ ))- )+ except Exception:+ _remove_windows_env_scope(mutations)+ raisereturn mutations@@ -151,15 +177,29 @@ def apply_mutations(manifest: DeploymentManifest) -> list[ManagedMutation]:"""Apply provider/user/system configuration for a deployment."""mutations: list[ManagedMutation] = []- if manifest.scope in {ConfigScope.USER.value, ConfigScope.SYSTEM.value}:- if os.name == "nt":- mutations.extend(_apply_windows_env_scope(manifest))- else:- mutations.extend(_apply_unix_env_scope(manifest))- mutations.extend(apply_provider_scope_mutations(manifest))+ manifest.mutations = mutations+ try:+ if manifest.scope in {ConfigScope.USER.value, ConfigScope.SYSTEM.value}:+ if os.name == "nt":+ mutations.extend(_apply_windows_env_scope(manifest))+ else:+ mutations.extend(_apply_unix_env_scope(manifest))+ # The registry already appends its records to manifest.mutations (this list).+ for mutation in apply_provider_scope_mutations(manifest):+ if not any(existing is mutation for existing in mutations):+ mutations.append(mutation)return mutations-- return [*mutations, *apply_provider_scope_mutations(manifest)]+ except Exception as exc:+ if mutations:+ try:+ revert_mutations(manifest)+ except Exception as rollback_exc:+ raise RuntimeError(+ f"mutation application failed: {exc}; rollback failed: {rollback_exc}"+ ) from exc+ else:+ manifest.mutations = []+ raisedef revert_mutations(manifest: DeploymentManifest) -> None:diff --git a/headroom/install/runtime.py b/headroom/install/runtime.pyindex ce2bc5424..3be93ff76 100644--- a/headroom/install/runtime.py+++ b/headroom/install/runtime.py@@ -3,6 +3,7 @@from __future__ import annotationsimport os+import shleximport shutilimport signalimport subprocess@@ -21,6 +22,9 @@ from .models import DeploymentManifest, InstallPreset, RuntimeKind, SupervisorKifrom .paths import log_path, pid_path, profile_rootfrom .state import load_manifest+_STOP_POLL_ATTEMPTS = 30+_STOP_POLL_DELAY = 0.1+# Inside the container the proxy must listen on every interface so the# host-side published port (127.0.0.1:<port>) can reach it. The proxy refuses# a non-loopback bind with no HEADROOM_PROXY_TOKEN unless it is acknowledged;@@ -213,6 +217,25 @@ def build_runtime_command(manifest: DeploymentManifest) -> list[str]:return command+def runtime_ownership(manifest: DeploymentManifest) -> str:+ """Classify the owner that must launch and supervise this runtime."""++ runtime_kind = getattr(manifest, "runtime_kind", None)+ if (+ runtime_kind == RuntimeKind.DOCKER.value+ or manifest.preset == InstallPreset.PERSISTENT_DOCKER.value+ ):+ return "docker-supervisor"+ if (+ sys.platform == "darwin"+ and manifest.preset == InstallPreset.PERSISTENT_SERVICE.value+ and manifest.runtime_kind == RuntimeKind.PYTHON.value+ and manifest.supervisor_kind == SupervisorKind.SERVICE.value+ ):+ return "launchd-exec"+ return "popen"++def _write_pid(profile: str, pid: int) -> None:path = pid_path(profile)path.parent.mkdir(parents=True, exist_ok=True)@@ -225,12 +248,154 @@ def _read_pid(profile: str) -> int | None:return Nonetry:return int(path.read_text().strip())+ except (OSError, ValueError):+ return None+++def _read_proc_metadata(pid: int) -> tuple[list[str], dict[str, str]] | None:+ """Read process identity without requiring the optional psutil package."""++ if Path(f"/proc/{pid}/cmdline").exists():+ try:+ cmdline = [+ part.decode(errors="replace")+ for part in Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0")+ if part+ ]+ environ = {+ part.split(b"=", 1)[0].decode(errors="replace"): part.split(b"=", 1)[1].decode(+ errors="replace"+ )+ for part in Path(f"/proc/{pid}/environ").read_bytes().split(b"\0")+ if b"=" in part+ }+ return cmdline, environ+ except (OSError, ValueError):+ return None++ if _is_windows():+ # WMI is part of Windows and exposes the full command line without+ # requiring psutil. Environment blocks are not exposed reliably, so+ # the command-line identity is checked together with the PID file.+ try:+ result = run(+ [+ "powershell.exe",+ "-NoProfile",+ "-NonInteractive",+ "-Command",+ f"(Get-CimInstance Win32_Process -Filter 'ProcessId = {pid}').CommandLine",+ ],+ capture_output=True,+ text=True,+ check=False,+ )+ except (OSError, subprocess.SubprocessError):+ return None+ raw = result.stdout.strip()+ if result.returncode != 0 or not raw:+ return None+ try:+ return shlex.split(raw, posix=False), {}+ except ValueError:+ return None++ # macOS does not expose /proc by default. `ps -wwE` is the supported+ # system interface that returns the complete command and inherited env.+ if sys.platform != "darwin":+ return None+ try:+ result = run(+ ["ps", "-wwE", "-p", str(pid), "-o", "command="],+ capture_output=True,+ text=True,+ check=False,+ )+ except (OSError, subprocess.SubprocessError):+ return None+ raw = result.stdout.strip()+ if result.returncode != 0 or not raw:+ return None+ try:+ fields = shlex.split(raw)except ValueError:return None+ environ = {}+ for field in fields:+ if field.startswith("HEADROOM_") and "=" in field:+ name, value = field.split("=", 1)+ environ[name] = value+ return fields, environ+++def _proxy_command_matches(cmdline: list[str], manifest: DeploymentManifest) -> bool:+ """Require the real Headroom proxy argv shape and configured port."""+ shape = any(+ cmdline[index : index + 3] == ["-m", "headroom.cli", "proxy"]+ or cmdline[index : index + 2] == ["headroom", "proxy"]+ for index in range(len(cmdline))+ )+ if not shape:+ return False+ for index, value in enumerate(cmdline):+ if value == "--port" and index + 1 < len(cmdline):+ return cmdline[index + 1] == str(manifest.port)+ if value.startswith("--port="):+ return value.split("=", 1)[1] == str(manifest.port)+ return False+++def _proxy_deployment_markers_match(cmdline: list[str], manifest: DeploymentManifest) -> bool:+ """Match the deployment identity embedded in supervisor-launched argv."""+ markers = {+ "--headroom-deployment-profile": manifest.profile,+ "--headroom-deployment-runtime": manifest.runtime_kind,+ }+ for name, expected in markers.items():+ if not any(+ cmdline[index : index + 2] == [name, str(expected)] or value == f"{name}={expected}"+ for index, value in enumerate(cmdline)+ ):+ return False+ return True-def _clear_pid(profile: str) -> None:+def _process_matches_runtime(pid: int, manifest: DeploymentManifest) -> bool:+ """Verify the PID's deployment identity before lifecycle operations."""+ return _process_identity(pid, manifest) is True+++def _process_identity(pid: int, manifest: DeploymentManifest) -> bool | None:+ """Return whether a PID matches, or ``None`` when identity is unavailable."""+ if pid == os.getpid():+ return False+ try:+ import psutil # type: ignore[import-untyped] # optional dependency++ process = psutil.Process(pid)+ cmdline = process.cmdline()+ environ = process.environ()+ except Exception:+ metadata = _read_proc_metadata(pid)+ if metadata is None:+ return None+ cmdline, environ = metadata+ if not _proxy_command_matches(cmdline, manifest):+ return False+ if _is_windows() and not environ:+ if not _proxy_deployment_markers_match(cmdline, manifest):+ return None+ return True+ return bool(+ environ.get("HEADROOM_DEPLOYMENT_PROFILE") == manifest.profile+ and environ.get("HEADROOM_DEPLOYMENT_RUNTIME") == manifest.runtime_kind+ )+++def _clear_pid(profile: str, *, expected_pid: int | None = None) -> None:path = pid_path(profile)+ if expected_pid is not None and _read_pid(profile) != expected_pid:+ returnif path.exists():path.unlink()@@ -291,12 +456,26 @@ def acquire_runtime_start_lock(profile: str) -> Iterator[bool]:def run_foreground(manifest: DeploymentManifest) -> int:"""Run the raw runtime command in the foreground."""+ if runtime_ownership(manifest) == "docker-supervisor":+ raise RuntimeError(+ "Docker deployments must be started by the Docker supervisor, not run_foreground"+ )command = build_runtime_command(manifest)env = _runtime_env(manifest)log_file_path = log_path(manifest.profile)log_file_path.parent.mkdir(parents=True, exist_ok=True)with open(log_file_path, "a", encoding="utf-8", errors="replace") as log_file:+ if runtime_ownership(manifest) == "launchd-exec":+ _write_pid(manifest.profile, os.getpid())+ try:+ os.dup2(log_file.fileno(), 1)+ os.dup2(log_file.fileno(), 2)+ os.execvpe(command[0], command, env)+ except BaseException:+ _clear_pid(manifest.profile, expected_pid=os.getpid())+ raise+proc = subprocess.Popen(command, env=env, stdout=log_file, stderr=log_file)_write_pid(manifest.profile, proc.pid)@@ -313,7 +492,7 @@ def run_foreground(manifest: DeploymentManifest) -> int:try:return proc.wait()finally:- _clear_pid(manifest.profile)+ _clear_pid(manifest.profile, expected_pid=proc.pid)def start_detached_agent(profile: str) -> subprocess.Popen[str]:@@ -359,51 +538,116 @@ def start_persistent_docker(manifest: DeploymentManifest) -> None:manifest.container_name,*command[5:], # drop initial `docker run --rm --name ...`]- run(+ remove = run(["docker", "rm", "-f", manifest.container_name],capture_output=True,text=True,)+ _require_docker_success(remove, "docker rm")subprocess.run(docker_cmd, check=True)def stop_runtime(manifest: DeploymentManifest) -> None:"""Stop the raw runtime for the deployment."""- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:- run(+ if runtime_ownership(manifest) == "docker-supervisor":+ stop = run(["docker", "stop", manifest.container_name],capture_output=True,text=True,)- run(+ _require_docker_success(stop, "docker stop")+ remove = run(["docker", "rm", "-f", manifest.container_name],capture_output=True,text=True,)+ _require_docker_success(remove, "docker rm")returnpid = _read_pid(manifest.profile)if pid is None:+ if pid_path(manifest.profile).exists():+ raise RuntimeError(+ f"Cannot stop deployment '{manifest.profile}': runner.pid is invalid"+ )+ return+ if not pid_alive(pid):+ _clear_pid(manifest.profile, expected_pid=pid)+ return+ identity = _process_identity(pid, manifest)+ if identity is None:+ raise RuntimeError(f"Cannot stop deployment '{manifest.profile}': runtime identity unknown")+ if not identity:+ _clear_pid(manifest.profile, expected_pid=pid)returntry:os.kill(pid, signal.SIGTERM)except (OSError, SystemError):# SystemError covers the Windows WinError 87 surfacing described in #1544.pass- _clear_pid(manifest.profile)+ for _ in range(_STOP_POLL_ATTEMPTS):+ if not pid_alive(pid):+ if not pid_alive(pid):+ _clear_pid(manifest.profile, expected_pid=pid)+ return+ raise RuntimeError(+ f"Cannot stop deployment '{manifest.profile}': runtime identity changed"+ )+ if _process_identity(pid, manifest) is not True:+ raise RuntimeError(+ f"Cannot stop deployment '{manifest.profile}': runtime identity changed"+ )+ time.sleep(_STOP_POLL_DELAY)+ raise RuntimeError(+ f"Cannot stop deployment '{manifest.profile}': runtime remained alive after SIGTERM"+ )+++def _command_output(result: Any) -> str:+ return str(+ getattr(result, "stderr", "") or getattr(result, "stdout", "") or "unknown error"+ ).strip()++def _require_docker_success(result: Any, operation: str) -> None:+ if result.returncode == 0:+ return+ if result.returncode == 1 and "no such container" in _command_output(result).lower():+ return+ raise RuntimeError(f"{operation} failed: {_command_output(result)}")-def wait_ready(manifest: DeploymentManifest, timeout_seconds: int = 30) -> bool:++def wait_ready(+ manifest: DeploymentManifest, timeout_seconds: int = 30, *, require_identity: bool = False+) -> bool:"""Wait for the deployment to report ready."""for _ in range(timeout_seconds):- if probe_ready(manifest.health_url):+ if probe_ready(manifest.health_url) and (+ not require_identity or runtime_status(manifest) == "running"+ ):return Truetime.sleep(1)return False+def runtime_ready(manifest: DeploymentManifest) -> bool:+ """Return ready only when health and deployment identity both match."""+ from .health import probe_ready as current_probe_ready++ if not hasattr(manifest, "runtime_kind"):+ return bool(getattr(manifest, "health_url", None)) and current_probe_ready(+ manifest.health_url+ )+ try:+ return runtime_status(manifest) == "running" and current_probe_ready(manifest.health_url)+ except AttributeError:+ # Test doubles and old third-party callers may provide only the health+ # contract; persisted DeploymentManifest instances always have identity.+ return False++def wait_stopped(manifest: DeploymentManifest, timeout_seconds: int = 15) -> bool:"""Wait for the deployment's health endpoint to stop answering."""@@ -418,7 +662,7 @@ def wait_stopped(manifest: DeploymentManifest, timeout_seconds: int = 15) -> boodef runtime_status(manifest: DeploymentManifest) -> str:"""Return a short status string for the deployment runtime."""- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:+ if runtime_ownership(manifest) == "docker-supervisor":result = run(["docker", "ps", "--format", "{{.Names}}"],capture_output=True,@@ -429,11 +673,22 @@ def runtime_status(manifest: DeploymentManifest) -> str:return "stopped"pid = _read_pid(manifest.profile)if pid is None:+ if pid_path(manifest.profile).exists():+ return "unknown"return "stopped"# Windows-safe liveness probe: a bare os.kill(pid, 0) here raised WinError 87# as a SystemError against the detached agent, crashing status and taking the# live proxy down with it (#1544).- return "running" if pid_alive(pid) else "stopped"+ if not pid_alive(pid):+ _clear_pid(manifest.profile, expected_pid=pid)+ return "stopped"+ identity = _process_identity(pid, manifest)+ if identity is None:+ return "unknown"+ if not identity:+ _clear_pid(manifest.profile, expected_pid=pid)+ return "stopped"+ return "running"def detect_current_deployment() -> tuple[DeploymentManifest | None, str]:@@ -456,7 +711,8 @@ def detect_current_deployment() -> tuple[DeploymentManifest | None, str]:if not profile:return None, "foreground"manifest = load_manifest(profile)- if preset == InstallPreset.PERSISTENT_DOCKER.value:+ runtime = os.environ.get("HEADROOM_DEPLOYMENT_RUNTIME")+ if preset == InstallPreset.PERSISTENT_DOCKER.value or runtime == RuntimeKind.DOCKER.value:return manifest, "docker"if manifest is None:return None, "foreground"diff --git a/headroom/install/state.py b/headroom/install/state.pyindex 16f66a897..5a091a87a 100644--- a/headroom/install/state.py+++ b/headroom/install/state.py@@ -12,7 +12,7 @@ from pathlib import Pathfrom typing import Anyfrom .models import ArtifactRecord, DeploymentManifest, ManagedMutation, iso_utc_now-from .paths import deploy_root, manifest_path, profile_root+from .paths import deploy_root, manifest_path, profile_root, recovery_manifest_pathlogger = logging.getLogger(__name__)@@ -34,7 +34,7 @@ def _atomic_write_text(path: Path, data: str) -> None:fd, tmp_name = tempfile.mkstemp(dir=directory, prefix=f".{path.name}.", suffix=".tmp")tmp_path = Path(tmp_name)try:- with os.fdopen(fd, "w", encoding="utf-8") as handle:+ with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:handle.write(data)handle.flush()os.fsync(handle.fileno())@@ -44,6 +44,22 @@ def _atomic_write_text(path: Path, data: str) -> None:raise+def _manifest_json(manifest: DeploymentManifest) -> str:+ return json.dumps(asdict(manifest), indent=2) + "\n"+++def _save_manifest_at(path: Path, manifest: DeploymentManifest, *, strict: bool) -> None:+ root = path.parent+ root.mkdir(parents=True, exist_ok=True)+ manifest.updated_at = iso_utc_now()+ try:+ _atomic_write_text(path, _manifest_json(manifest))+ except OSError:+ if strict:+ raise+ logger.warning("Cannot save deployment manifest: %s — continuing without persistence", path)++def save_manifest(manifest: DeploymentManifest) -> None:"""Persist a deployment manifest to disk.@@ -52,15 +68,29 @@ def save_manifest(manifest: DeploymentManifest) -> None:read-only filesystems by logging a warning instead of crashing."""try:- root = profile_root(manifest.profile)- root.mkdir(parents=True, exist_ok=True)- manifest.updated_at = iso_utc_now()- path = manifest_path(manifest.profile)- _atomic_write_text(path, json.dumps(asdict(manifest), indent=2) + "\n")+ _save_manifest_at(manifest_path(manifest.profile), manifest, strict=False)except OSError as e:logger.warning("Cannot save deployment manifest: %s — continuing without persistence", e)+def save_manifest_strict(manifest: DeploymentManifest) -> None:+ """Persist a manifest and propagate errors when recovery depends on it."""++ _save_manifest_at(manifest_path(manifest.profile), manifest, strict=True)+++def save_recovery_manifest(manifest: DeploymentManifest) -> None:+ """Persist an inactive, complete snapshot for failed deployment recovery."""++ _save_manifest_at(recovery_manifest_path(manifest.profile), manifest, strict=True)+++def delete_recovery_manifest(profile: str = "default") -> None:+ """Delete an inactive recovery snapshot when its transaction is complete."""++ recovery_manifest_path(profile).unlink(missing_ok=True)++# The Docker image org moved from a personal repo to the project org. The old# ``ghcr.io/chopratejas/headroom`` repo is frozen at 0.27.0, so a manifest that# still pins it silently runs ~5 minor versions behind the CLI with no drift@@ -127,4 +157,4 @@ def delete_manifest(profile: str = "default") -> None:root = profile_root(profile)if root.exists():- shutil.rmtree(root, ignore_errors=True)+ shutil.rmtree(root)diff --git a/headroom/install/supervisors.py b/headroom/install/supervisors.pyindex 7591c1b18..b0db7fe57 100644--- a/headroom/install/supervisors.py+++ b/headroom/install/supervisors.py@@ -445,7 +445,7 @@ def _cleanup_windows_tasks_for_fallback(names: list[str], *, must_delete: set[st)-def install_supervisor(manifest: DeploymentManifest) -> list[ArtifactRecord]:+def install_supervisor(manifest: DeploymentManifest, *, start: bool = True) -> list[ArtifactRecord]:"""Install service/task artifacts for the deployment."""records = render_runner_scripts(manifest)@@ -522,7 +522,8 @@ def install_supervisor(manifest: DeploymentManifest) -> list[ArtifactRecord]:and manifest.supervisor_kind == SupervisorKind.SERVICE.valueelse f"gui/{os.getuid()}")- _bootstrap_with_retry(bootstrap_domain, plist_path)+ if start:+ _bootstrap_with_retry(bootstrap_domain, plist_path)records.append(ArtifactRecord(kind="plist", path=str(plist_path)))return records@@ -661,6 +662,8 @@ def stop_supervisor(manifest: DeploymentManifest) -> None:if manifest.supervisor_kind == SupervisorKind.NONE.value:returnif sys.platform.startswith("linux"):+ if manifest.supervisor_kind == SupervisorKind.TASK.value:+ returnflags = [] if manifest.scope == "system" else ["--user"]subprocess.run(["systemctl", *flags, "stop", manifest.service_name], check=True)return@@ -701,19 +704,21 @@ def remove_supervisor(manifest: DeploymentManifest) -> None:if sys.platform.startswith("linux"):if manifest.supervisor_kind == SupervisorKind.SERVICE.value:flags = [] if manifest.scope == "system" else ["--user"]- run(+ result = run(["systemctl", *flags, "disable", "--now", manifest.service_name],capture_output=True,text=True,)+ _require_removal_success(result, "systemctl disable --now", absent_text="not loaded")unit_path, _ = _linux_service_unit(manifest, unix_run_script_path(manifest.profile))if unit_path.exists():unit_path.unlink()- run(+ result = run(["systemctl", *flags, "daemon-reload"],capture_output=True,text=True,)+ _require_removal_success(result, "systemctl daemon-reload")returncron_path, _ = _linux_task_spec(manifest, unix_ensure_script_path(manifest.profile))if cron_path and cron_path.exists():@@ -725,17 +730,20 @@ def remove_supervisor(manifest: DeploymentManifest) -> None:text=True,)if current.returncode != 0:+ if "no crontab for" not in _result_text(current).lower():+ raise click.ClickException(f"crontab -l failed: {_result_text(current)}")returnmarker_start = f"# >>> headroom {manifest.profile} >>>"marker_end = f"# <<< headroom {manifest.profile} <<<"pattern = re.compile(re.escape(marker_start) + r".*?" + re.escape(marker_end), re.DOTALL)content = pattern.sub("", current.stdout).strip()- run(+ result = run(["crontab", "-"],input=(content + "\n") if content else "",text=True,check=True,)+ _require_removal_success(result, "crontab update")returnif sys.platform == "darwin":@@ -753,27 +761,64 @@ def remove_supervisor(manifest: DeploymentManifest) -> None:and manifest.supervisor_kind == SupervisorKind.SERVICE.valueelse f"gui/{os.getuid()}")- run(+ result = run(["launchctl", "bootout", f"{domain}/{label}"],capture_output=True,text=True,)+ _require_removal_success(result, "launchctl bootout", absent_codes={_LAUNCHCTL_ESRCH})if plist_path.exists():plist_path.unlink()returnif _is_windows():if manifest.supervisor_kind == SupervisorKind.SERVICE.value:- run(+ result = run(["sc.exe", "stop", manifest.service_name],capture_output=True,text=True,)- run(+ _require_removal_success(result, "sc.exe stop", absent_text="does not exist")+ result = run(["sc.exe", "delete", manifest.service_name],capture_output=True,text=True,)+ _require_removal_success(result, "sc.exe delete", absent_text="does not exist")return- _delete_windows_task(f"{manifest.service_name}-startup")- _delete_windows_task(f"{manifest.service_name}-health")+ result = run(+ ["schtasks", "/Delete", "/TN", f"{manifest.service_name}-startup", "/F"],+ capture_output=True,+ text=True,+ )+ _require_removal_success(result, "schtasks startup delete", absent_text="does not exist")+ result = run(+ ["schtasks", "/Delete", "/TN", f"{manifest.service_name}-health", "/F"],+ capture_output=True,+ text=True,+ )+ _require_removal_success(result, "schtasks health delete", absent_text="does not exist")+++def _result_text(result: object) -> str:+ return str(+ getattr(result, "stderr", "") or getattr(result, "stdout", "") or "unknown error"+ ).strip()+++def _require_removal_success(+ result: object,+ operation: str,+ *,+ absent_codes: set[int] | None = None,+ absent_text: str | None = None,+) -> None:+ returncode = int(getattr(result, "returncode", 0))+ detail = _result_text(result)+ if returncode == 0:+ return+ if absent_codes and returncode in absent_codes:+ return+ if absent_text and absent_text in detail.lower():+ return+ raise click.ClickException(f"{operation} failed: {detail}")diff --git a/headroom/providers/install_registry.py b/headroom/providers/install_registry.pyindex a3e053aa1..a998bed90 100644--- a/headroom/providers/install_registry.py+++ b/headroom/providers/install_registry.py@@ -85,13 +85,33 @@ def build_install_target_envs(def apply_provider_scope_mutations(manifest: DeploymentManifest) -> list[ManagedMutation]:"""Apply provider-scope mutations owned by provider slices."""mutations: list[ManagedMutation] = []- for target in manifest.targets:- handlers = _PROVIDER_SCOPE_HANDLERS.get(target)- if handlers is None:- continue- mutation = handlers[0](manifest)- if mutation is not None:- mutations.append(mutation)+ tracked = getattr(manifest, "mutations", None)+ try:+ for target in manifest.targets:+ handlers = _PROVIDER_SCOPE_HANDLERS.get(target)+ if handlers is None:+ continue+ mutation = handlers[0](manifest)+ if mutation is not None:+ mutations.append(mutation)+ if tracked is not None and tracked is not mutations:+ tracked.append(mutation)+ except Exception as exc:+ rollback_errors: list[Exception] = []+ for mutation in reversed(mutations):+ try:+ revert_provider_scope_mutation(manifest, mutation)+ except Exception as rollback_exc:+ rollback_errors.append(rollback_exc)+ else:+ if tracked is not None and mutation in tracked:+ tracked.remove(mutation)+ if rollback_errors:+ details = "; ".join(str(error) for error in rollback_errors)+ raise RuntimeError(+ f"provider mutation failed: {exc}; rollback failed: {details}"+ ) from exc+ raisereturn mutationsdiff --git a/tests/test_cli/test_install_cli.py b/tests/test_cli/test_install_cli.pyindex 1fd12c251..c392d6b82 100644--- a/tests/test_cli/test_install_cli.py+++ b/tests/test_cli/test_install_cli.py@@ -1,5 +1,7 @@from __future__ import annotations+import json+from pathlib import Pathfrom types import SimpleNamespaceimport click@@ -8,7 +10,9 @@ from click.testing import CliRunnerfrom headroom.cli import install as instfrom headroom.cli.main import main+from headroom.install.models import DeploymentManifest, ManagedMutationfrom headroom.install.planner import build_tool_envs+from headroom.install.state import load_manifest as load_state_manifestdef test_require_manifest_resolves_single_profile_when_default_missing(monkeypatch):@@ -148,7 +152,7 @@ def test_install_apply_starts_service_supervisor(monkeypatch) -> None:"headroom.cli.install.apply_mutations",lambda deployment: calls.append("apply") or [],)- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: calls.append("save"))@@ -163,7 +167,7 @@ def test_install_apply_starts_service_supervisor(monkeypatch) -> None:lambda deployment: calls.append("start_docker"),)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")@@ -173,7 +177,7 @@ def test_install_apply_starts_service_supervisor(monkeypatch) -> None:assert result.exit_code == 0, result.outputassert "Installed persistent deployment 'default'" in result.outputassert "Targets: claude, codex" in result.output- assert calls == ["save", "start_service", "apply", "save"]+ assert calls == ["save", "save", "start_service", "apply", "save"]def test_install_apply_announces_windows_service_fallback(monkeypatch) -> None:@@ -197,7 +201,7 @@ def test_install_apply_announces_windows_service_fallback(monkeypatch) -> None:monkeypatch.setattr("headroom.cli.install._is_windows", lambda: True)monkeypatch.setattr("headroom.cli.install.build_manifest", lambda **_: Manifest())monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)@@ -206,7 +210,7 @@ def test_install_apply_announces_windows_service_fallback(monkeypatch) -> None:"headroom.cli.install.start_detached_agent", lambda profile: calls.append("start_agent"))monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)result = runner.invoke(main, ["install", "apply", "--preset", "persistent-service"])@@ -246,12 +250,12 @@ def test_install_apply_forwards_no_http2_to_build_manifest(monkeypatch) -> None:monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build_manifest)monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.start_detached_agent", lambda profile: None)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)result = runner.invoke(main, ["install", "apply", "--no-http2"])@@ -284,12 +288,12 @@ def _patch_apply_pipeline(monkeypatch, captured: dict[str, object]):monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build_manifest)monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.start_detached_agent", lambda profile: None)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)@@ -409,12 +413,12 @@ def _apply_capturing_build_manifest(monkeypatch) -> dict[str, object]:monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build_manifest)monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.start_detached_agent", lambda profile: None)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)return captured@@ -461,7 +465,10 @@ def test_install_status_includes_backend_from_health_probe(monkeypatch) -> None:health_url = "http://127.0.0.1:8787/readyz"monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())- monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")+ monkeypatch.setattr(+ "headroom.cli.install.runtime_status",+ lambda manifest: "running",+ )monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: True)monkeypatch.setattr("headroom.cli.install.probe_json",@@ -536,7 +543,7 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor"))monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45, **kwargs: True)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda manifest: calls.append("apply") or []@@ -561,6 +568,100 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:][email protected]("supervisor_kind", ["service", "task"])+def test_stop_deployment_stops_external_supervisor_before_docker(+ monkeypatch, supervisor_kind: str+) -> None:+ calls: list[str] = []+ manifest = SimpleNamespace(+ profile="default",+ preset="persistent-service",+ runtime_kind="docker",+ supervisor_kind=supervisor_kind,+ health_url="http://127.0.0.1:8787/readyz",+ )+ monkeypatch.setattr(inst, "stop_supervisor", lambda current: calls.append("supervisor"))+ monkeypatch.setattr(inst, "stop_runtime", lambda current: calls.append("runtime"))+ monkeypatch.setattr(inst, "wait_stopped", lambda current: calls.append("wait_stopped") or True)++ inst._stop_deployment(manifest)++ assert calls == ["supervisor", "runtime", "wait_stopped"]+++def test_stop_deployment_stops_docker_even_when_supervisor_stop_fails(monkeypatch) -> None:+ calls: list[str] = []+ manifest = SimpleNamespace(+ profile="default",+ preset="persistent-service",+ runtime_kind="docker",+ supervisor_kind="service",+ )++ def fail_supervisor(current):+ calls.append("supervisor")+ raise RuntimeError("supervisor unavailable")++ monkeypatch.setattr(inst, "stop_supervisor", fail_supervisor)+ monkeypatch.setattr(inst, "stop_runtime", lambda current: calls.append("runtime"))++ with pytest.raises(RuntimeError, match="supervisor unavailable"):+ inst._stop_deployment(manifest)+ assert calls == ["supervisor", "runtime"]+++def test_remove_deployment_retains_manifest_when_supervisor_stop_fails(monkeypatch) -> None:+ calls: list[str] = []+ manifest = SimpleNamespace(+ profile="default",+ preset="persistent-service",+ runtime_kind="docker",+ supervisor_kind="service",+ mutations=[],+ )+ monkeypatch.setattr(+ inst,+ "stop_supervisor",+ lambda current: (_ for _ in ()).throw(RuntimeError("supervisor unavailable")),+ )+ monkeypatch.setattr(inst, "stop_runtime", lambda current: calls.append("runtime"))+ monkeypatch.setattr(+ inst, "remove_supervisor", lambda current: calls.append("remove-supervisor")+ )+ monkeypatch.setattr(inst, "delete_manifest", lambda profile: calls.append("delete"))++ with pytest.raises(RuntimeError, match="supervisor unavailable"):+ inst._remove_deployment(manifest)+ assert calls == ["runtime", "remove-supervisor"]+++def test_remove_deployment_reports_all_cleanup_failures(monkeypatch) -> None:+ manifest = SimpleNamespace(profile="default", mutations=[object()])+ monkeypatch.setattr(+ inst,+ "_deactivate_deployment_mutations",+ lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("mutation failed")),+ )+ monkeypatch.setattr(+ inst,+ "_stop_deployment",+ lambda current: (_ for _ in ()).throw(RuntimeError("runtime failed")),+ )+ monkeypatch.setattr(+ inst,+ "remove_supervisor",+ lambda current: (_ for _ in ()).throw(RuntimeError("supervisor failed")),+ )++ with pytest.raises(RuntimeError) as exc:+ inst._remove_deployment(manifest)++ message = str(exc.value)+ assert all(+ item in message for item in ("mutation failed", "runtime failed", "supervisor failed")+ )++def _restart_race_manifest():class Manifest:profile = "default"@@ -602,7 +703,8 @@ def test_install_restart_waits_for_old_process_before_starting(monkeypatch) -> N"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor"))monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45: True+ "headroom.cli.install.wait_ready",+ lambda manifest, timeout_seconds=45, **kwargs: True,)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda manifest: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: None)@@ -659,6 +761,7 @@ def test_install_start_noops_when_already_healthy(monkeypatch) -> None:mutations = [object()]monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: True)monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")@@ -671,6 +774,28 @@ def test_install_start_noops_when_already_healthy(monkeypatch) -> None:assert calls == []+def test_start_deployment_requires_identity_for_post_start_readiness(monkeypatch) -> None:+ manifest = SimpleNamespace(+ profile="default",+ preset="persistent-task",+ runtime_kind="python",+ supervisor_kind="none",+ health_url="http://127.0.0.1:8787/readyz",+ )+ waits: list[dict[str, object]] = []+ monkeypatch.setattr(inst, "runtime_status", lambda current: "stopped")+ monkeypatch.setattr(inst, "start_detached_agent", lambda profile: None)+ monkeypatch.setattr(+ inst,+ "wait_ready",+ lambda current, timeout_seconds, **kwargs: waits.append(kwargs) or True,+ )++ inst._start_deployment(manifest, assume_start_lock=True)++ assert waits == [{"require_identity": True}]++def test_install_start_noops_for_healthy_docker_without_docker_on_path(monkeypatch) -> None:runner = CliRunner()@@ -688,13 +813,14 @@ def test_install_start_noops_for_healthy_docker_without_docker_on_path(monkeypatmutations = [object()]monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: True)monkeypatch.setattr("headroom.cli.install.shutil.which", lambda name, *args, **kwargs: None)result = runner.invoke(main, ["install", "start"])- assert result.exit_code == 0, result.output- assert "Started deployment 'default'." in result.output+ assert result.exit_code != 0+ assert "docker' was not found on PATH" in result.outputdef test_install_start_does_not_spawn_when_start_lock_is_contended(monkeypatch) -> None:@@ -763,7 +889,8 @@ def test_install_start_restarts_wedged_runtime_under_single_lock(monkeypatch) ->monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")wait_results = iter([False, True])monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds: next(wait_results)+ "headroom.cli.install.wait_ready",+ lambda manifest, timeout_seconds, **kwargs: next(wait_results),)monkeypatch.setattr("headroom.cli.install.revert_mutations", lambda manifest: calls.append("revert")@@ -831,12 +958,12 @@ def test_install_apply_accepts_opencode_target(monkeypatch) -> None:monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build_manifest)monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.start_detached_agent", lambda profile: None)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)result = runner.invoke(@@ -891,7 +1018,9 @@ def test_install_apply_restores_previous_deployment_after_failed_update(monkeypa)monkeypatch.setattr("headroom.cli.install.install_supervisor",- lambda deployment: calls.append(f"supervisor:{','.join(deployment.targets)}") or [],+ lambda deployment, **kwargs: (+ calls.append(f"supervisor:{','.join(deployment.targets)}") or []+ ),)monkeypatch.setattr("headroom.cli.install.save_manifest",@@ -917,6 +1046,10 @@ def test_install_apply_restores_previous_deployment_after_failed_update(monkeypa"headroom.cli.install.delete_manifest",lambda profile: calls.append(f"delete:{profile}"),)+ monkeypatch.setattr(+ "headroom.cli.install.delete_recovery_manifest",+ lambda profile: calls.append(f"delete-recovery:{profile}"),+ )def _start(deployment) -> None:calls.append(f"start:{','.join(deployment.targets)}")@@ -935,6 +1068,7 @@ def test_install_apply_restores_previous_deployment_after_failed_update(monkeypa"stop-runtime:codex","remove-supervisor:codex","delete:default",+ "save:claude","supervisor:claude","save:claude","start:claude",@@ -947,9 +1081,399 @@ def test_install_apply_restores_previous_deployment_after_failed_update(monkeypa"start:codex","apply:codex","save:codex",+ "delete-recovery:default",]+def test_install_apply_reports_restore_failure_with_recovery_path(monkeypatch) -> None:+ previous = SimpleNamespace(profile="default")+ new = SimpleNamespace(profile="default", artifacts=[], mutations=[])+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)+ monkeypatch.setattr(inst, "_save_recovery_snapshot", lambda *args: None)+ monkeypatch.setattr(inst, "_remove_deployment", lambda manifest: None)+ monkeypatch.setattr(inst, "_save_apply_manifest", lambda manifest: None)+ monkeypatch.setattr(inst, "install_supervisor", lambda manifest, **kwargs: [])+ monkeypatch.setattr(+ inst,+ "_start_deployment",+ lambda manifest: (_ for _ in ()).throw(click.ClickException("startup failed")),+ )+ monkeypatch.setattr(+ inst,+ "_restore_deployment",+ lambda manifest: (_ for _ in ()).throw(RuntimeError("restore failed")),+ )++ with pytest.raises(click.ClickException) as exc:+ inst._apply_manifest(new)++ message = str(exc.value)+ assert "restore failed" in message+ assert "recovery snapshot:" in message+ assert "restore it after resolving the failure" in message+++def test_install_apply_old_removal_failure_is_actionable_and_does_not_start_new(+ monkeypatch,+) -> None:+ previous = SimpleNamespace(profile="default")+ new = SimpleNamespace(profile="default", artifacts=[], mutations=[])+ calls: list[str] = []+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)+ monkeypatch.setattr(inst, "_save_recovery_snapshot", lambda *args: calls.append("snapshot"))+ monkeypatch.setattr(+ inst,+ "_remove_deployment",+ lambda manifest: (_ for _ in ()).throw(RuntimeError("old removal failed")),+ )+ monkeypatch.setattr(inst, "_start_deployment", lambda manifest: calls.append("start"))++ with pytest.raises(click.ClickException) as exc:+ inst._apply_manifest(new)++ assert "old removal failed" in str(exc.value)+ assert "no new owner was started" in str(exc.value)+ assert calls == ["snapshot"]+++def test_install_apply_uses_requested_profile_for_recovery_snapshot(+ monkeypatch, tmp_path: Path+) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ previous = DeploymentManifest(+ profile="loaded-profile",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=["old"],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ new = DeploymentManifest(+ profile="requested-profile",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=["new"],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)+ monkeypatch.setattr(inst, "_remove_deployment", lambda manifest: None)+ monkeypatch.setattr(inst, "_save_apply_manifest", lambda manifest: None)+ monkeypatch.setattr(inst, "install_supervisor", lambda manifest, **kwargs: [])+ monkeypatch.setattr(+ inst,+ "_start_deployment",+ lambda manifest: (_ for _ in ()).throw(click.ClickException("startup failed")),+ )++ with pytest.raises(click.ClickException):+ inst._apply_manifest(new)++ assert (tmp_path / ".headroom" / "deploy" / "requested-profile.recovery.json").exists()+ assert not (tmp_path / ".headroom" / "deploy" / "loaded-profile.recovery.json").exists()+++def test_restore_deployment_requires_durable_manifest_before_start(monkeypatch) -> None:+ manifest = DeploymentManifest(+ profile="default",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=[],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ calls: list[str] = []+ monkeypatch.setattr(inst, "install_supervisor", lambda current, **kwargs: [])+ monkeypatch.setattr(+ inst,+ "_save_apply_manifest",+ lambda current: (_ for _ in ()).throw(OSError("restore manifest busy")),+ )+ monkeypatch.setattr(inst, "_start_deployment", lambda current: calls.append("start"))++ with pytest.raises(OSError, match="restore manifest busy"):+ inst._restore_deployment(manifest)+ assert calls == []+++def test_apply_persists_before_darwin_bootstrap_or_start_on_save_failure(monkeypatch) -> None:+ manifest = DeploymentManifest(+ profile="default",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=[],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ install_calls: list[bool] = []+ bootstrap_calls: list[str] = []+ start_calls: list[str] = []+ save_calls = 0++ def save(current) -> None:+ nonlocal save_calls+ save_calls += 1+ if save_calls == 2:+ raise OSError("active manifest busy")++ def install(current, *, start=True):+ install_calls.append(start)+ if start:+ bootstrap_calls.append(current.profile)+ return []++ monkeypatch.setattr(inst, "load_manifest", lambda profile: None)+ monkeypatch.setattr(inst, "_save_apply_manifest", save)+ monkeypatch.setattr(inst, "install_supervisor", install)+ monkeypatch.setattr(+ inst, "_start_deployment", lambda current: start_calls.append(current.profile)+ )+ monkeypatch.setattr(inst, "_remove_deployment", lambda current: None)++ with pytest.raises(click.ClickException, match="active manifest busy"):+ inst._apply_manifest(manifest)++ assert install_calls == [False]+ assert bootstrap_calls == []+ assert start_calls == []+++def test_activate_mutations_reverts_side_effects_when_strict_save_fails(monkeypatch) -> None:+ manifest = DeploymentManifest(+ profile="default",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=[],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ mutation = ManagedMutation(target="env", kind="shell-block", path="settings")+ reverted: list[ManagedMutation] = []+ monkeypatch.setattr(inst, "apply_mutations", lambda current: [mutation])+ monkeypatch.setattr(+ inst,+ "_save_apply_manifest",+ lambda current: (_ for _ in ()).throw(OSError("manifest busy")),+ )+ monkeypatch.setattr(+ inst, "revert_mutations", lambda current: reverted.extend(current.mutations)+ )++ with pytest.raises(OSError, match="manifest busy"):+ inst._activate_deployment_mutations(manifest)+ assert reverted == [mutation]+ assert manifest.mutations == []+++def test_recovery_snapshot_delete_failure_is_actionable(monkeypatch) -> None:+ monkeypatch.setattr(+ inst,+ "delete_recovery_manifest",+ lambda profile: (_ for _ in ()).throw(OSError("snapshot busy")),+ )++ with pytest.raises(click.ClickException, match="could not be deleted") as exc:+ inst._delete_recovery_snapshot("default")+ assert "snapshot busy" in str(exc.value)+ assert "retained" in str(exc.value)+++def test_install_apply_keeps_new_owner_recovery_fail_closed(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ previous = DeploymentManifest(+ profile="default",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=["old"],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ new = DeploymentManifest(+ profile="default",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=["new"],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ calls: list[str] = []++ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)+ real_save_recovery = inst._save_recovery_snapshot++ def save_recovery(manifest, profile):+ real_save_recovery(manifest, profile)+ calls.append("snapshot")++ monkeypatch.setattr(inst, "_save_recovery_snapshot", save_recovery)++ def remove(manifest):+ calls.append("remove-new" if manifest is new else "remove-old")+ if manifest is new:+ raise RuntimeError("new deployment cleanup failed")++ monkeypatch.setattr(inst, "_remove_deployment", remove)+ real_save_apply = inst._save_apply_manifest++ def save_apply(manifest):+ real_save_apply(manifest)+ calls.append("save-new")++ monkeypatch.setattr(inst, "_save_apply_manifest", save_apply)+ monkeypatch.setattr(+ inst, "install_supervisor", lambda manifest, **kwargs: calls.append("install-new") or []+ )+ monkeypatch.setattr(+ inst,+ "_start_deployment",+ lambda manifest: (_ for _ in ()).throw(click.ClickException("startup failed")),+ )+ monkeypatch.setattr(inst, "_restore_deployment", lambda manifest: calls.append("restore-old"))+ monkeypatch.setattr(+ inst, "delete_recovery_manifest", lambda profile: calls.append("delete-snapshot")+ )++ with pytest.raises(click.ClickException) as exc:+ inst._apply_manifest(new)++ message = str(exc.value)+ assert "startup failed" in message+ assert "new deployment cleanup failed" in message+ assert "recovery snapshot" in message+ assert "remove the new owner before restoring the snapshot" in message+ assert calls == ["snapshot", "remove-old", "save-new", "install-new", "save-new", "remove-new"]+ assert load_state_manifest("default") is not None+ recovery = tmp_path / ".headroom" / "deploy" / "default.recovery.json"+ assert json.loads(recovery.read_text(encoding="utf-8"))["targets"] == ["old"]+++def test_install_apply_keeps_snapshot_when_active_persistence_fails(+ monkeypatch, tmp_path: Path+) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ previous = DeploymentManifest(+ profile="default",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=["old"],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ new = DeploymentManifest(+ profile="default",+ preset="persistent-service",+ runtime_kind="python",+ supervisor_kind="service",+ scope="user",+ provider_mode="manual",+ targets=["new"],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ )+ calls: list[str] = []+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)+ monkeypatch.setattr(inst, "_remove_deployment", lambda manifest: calls.append("cleanup"))+ monkeypatch.setattr(inst, "install_supervisor", lambda manifest, **kwargs: [])+ saves = 0++ def fail_active_save(manifest):+ nonlocal saves+ saves += 1+ if saves == 1:+ inst.save_manifest_strict(manifest)+ return+ raise OSError("active manifest busy")++ monkeypatch.setattr(inst, "_save_apply_manifest", fail_active_save)+ monkeypatch.setattr(inst, "_restore_deployment", lambda manifest: calls.append("restore"))+ monkeypatch.setattr(inst, "delete_recovery_manifest", lambda profile: calls.append("delete"))++ with pytest.raises(click.ClickException) as exc:+ inst._apply_manifest(new)++ assert "active manifest persistence failed" in str(exc.value)+ assert "recovery snapshot:" in str(exc.value)+ assert calls == ["cleanup", "cleanup"]+ assert load_state_manifest("default") is not None+ assert (tmp_path / ".headroom" / "deploy" / "default.recovery.json").exists()+++def test_install_apply_persists_new_manifest_before_supervisor_install(monkeypatch) -> None:+ new = SimpleNamespace(profile="default", artifacts=[], mutations=[])+ calls: list[str] = []++ monkeypatch.setattr(inst, "load_manifest", lambda profile: None)+ monkeypatch.setattr(inst, "_save_apply_manifest", lambda manifest: calls.append("save"))+ monkeypatch.setattr(+ inst,+ "install_supervisor",+ lambda manifest, **kwargs: (_ for _ in ()).throw(RuntimeError("install failed")),+ )+ monkeypatch.setattr(inst, "_remove_deployment", lambda manifest: calls.append("cleanup"))++ with pytest.raises(click.ClickException, match="install failed"):+ inst._apply_manifest(new)++ assert calls == ["save", "cleanup"]+++def test_install_apply_without_previous_has_no_recovery_guidance(monkeypatch) -> None:+ new = SimpleNamespace(profile="default", artifacts=[], mutations=[])+ monkeypatch.setattr(inst, "load_manifest", lambda profile: None)+ monkeypatch.setattr(inst, "_save_apply_manifest", lambda manifest: None)+ monkeypatch.setattr(inst, "install_supervisor", lambda manifest, **kwargs: [])+ monkeypatch.setattr(+ inst,+ "_start_deployment",+ lambda manifest: (_ for _ in ()).throw(click.ClickException("startup failed")),+ )+ monkeypatch.setattr(+ inst,+ "_remove_deployment",+ lambda manifest: (_ for _ in ()).throw(RuntimeError("cleanup failed")),+ )++ with pytest.raises(click.ClickException) as exc:+ inst._apply_manifest(new)++ message = str(exc.value)+ assert "startup failed" in message and "cleanup failed" in message+ assert "recovery snapshot" not in message++def test_install_start_rejects_task_lifecycle(monkeypatch) -> None:runner = CliRunner()@@ -995,7 +1519,7 @@ def test_install_apply_uses_docker_runtime_for_persistent_docker(monkeypatch) ->monkeypatch.setattr("headroom.cli.install.build_manifest", lambda **_: Manifest())monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)monkeypatch.setattr("headroom.cli.install.runtime_status", lambda deployment: "stopped")@@ -1012,7 +1536,7 @@ def test_install_apply_uses_docker_runtime_for_persistent_docker(monkeypatch) ->lambda deployment: calls.append("start_docker"),)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)monkeypatch.setattr("headroom.cli.install.runtime_status", lambda deployment: "stopped")@@ -1065,7 +1589,7 @@ def test_deploy_prefers_docker_when_available(monkeypatch) -> None:monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build)monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)monkeypatch.setattr("headroom.cli.install.runtime_status", lambda deployment: "stopped")@@ -1082,7 +1606,7 @@ def test_deploy_prefers_docker_when_available(monkeypatch) -> None:lambda deployment: calls.append("start_docker"),)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)result = runner.invoke(main, ["deploy"])@@ -1129,7 +1653,7 @@ def test_deploy_prefers_gpu_docker_when_available(monkeypatch) -> None:monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build)monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)monkeypatch.setattr("headroom.cli.install.runtime_status", lambda deployment: "stopped")@@ -1143,7 +1667,7 @@ def test_deploy_prefers_gpu_docker_when_available(monkeypatch) -> None:monkeypatch.setattr("headroom.cli.install.acquire_runtime_start_lock", fake_lock)monkeypatch.setattr("headroom.cli.install.start_persistent_docker", lambda deployment: None)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)result = runner.invoke(main, ["deploy"])@@ -1181,7 +1705,7 @@ def test_deploy_falls_back_to_detached_python_without_supervisor(monkeypatch) ->monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])monkeypatch.setattr("headroom.cli.install.install_supervisor",- lambda deployment: calls.append(f"supervisor:{deployment.supervisor_kind}") or [],+ lambda deployment, **kwargs: calls.append(f"supervisor:{deployment.supervisor_kind}") or [],)monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)@@ -1199,7 +1723,7 @@ def test_deploy_falls_back_to_detached_python_without_supervisor(monkeypatch) ->lambda profile: calls.append(f"agent:{profile}"),)monkeypatch.setattr(- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True)result = runner.invoke(main, ["deploy", "--no-docker"])@@ -1210,7 +1734,7 @@ def test_deploy_falls_back_to_detached_python_without_supervisor(monkeypatch) ->assert calls == ["supervisor:none", "agent:default"]-def test_install_remove_continues_when_runtime_teardown_errors(monkeypatch) -> None:+def test_install_remove_retains_manifest_when_runtime_teardown_errors(monkeypatch) -> None:runner = CliRunner()calls: list[str] = []@@ -1248,8 +1772,10 @@ def test_install_remove_continues_when_runtime_teardown_errors(monkeypatch) -> Nresult = runner.invoke(main, ["install", "remove"])- assert result.exit_code == 0, result.output- assert calls == ["revert", "remove_supervisor", "delete"]+ assert result.exit_code != 0+ assert "Error: Failed to remove deployment 'default': cleanup failed:" in result.output+ assert "Traceback" not in result.output+ assert calls == ["revert", "remove_supervisor"]def test_install_agent_ensure_reports_already_healthy(monkeypatch) -> None:@@ -1262,8 +1788,10 @@ def test_install_agent_ensure_reports_already_healthy(monkeypatch) -> None:tool_envs = {}profile = "default"health_url = "http://127.0.0.1:8787/readyz"+ mutations = []monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: True)result = runner.invoke(main, ["install", "agent", "ensure"])@@ -1305,6 +1833,7 @@ def test_install_agent_ensure_no_spawn_when_lock_not_acquired(monkeypatch) -> Nohealth_url = "http://127.0.0.1:8787/readyz"monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)import contextlib@@ -1355,7 +1884,13 @@ def test_install_agent_ensure_stops_wedged_runtime_before_restart(monkeypatch) -monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")- monkeypatch.setattr("headroom.cli.install.wait_ready", lambda manifest, timeout_seconds: False)+ wait_calls: list[dict[str, object]] = []++ def fake_wait_ready(manifest, timeout_seconds, **kwargs):+ wait_calls.append(kwargs)+ return False++ monkeypatch.setattr("headroom.cli.install.wait_ready", fake_wait_ready)monkeypatch.setattr("headroom.cli.install.revert_mutations", lambda manifest: calls.append("revert"))@@ -1391,6 +1926,7 @@ def test_install_agent_ensure_stops_wedged_runtime_before_restart(monkeypatch) -assert calls.index("revert") < calls.index("stop")assert calls.index("stop") < calls.index("start_deployment")assert calls.index("start_deployment") < calls.index("apply")+ assert wait_calls == [{"require_identity": True}]assert "start_agent" not in callsassert "start_docker" not in calls@@ -1435,7 +1971,9 @@ def test_install_agent_ensure_starts_when_stopped_and_lock_acquired(monkeypatch)yield Truemonkeypatch.setattr("headroom.cli.install.acquire_runtime_start_lock", fake_lock)- monkeypatch.setattr("headroom.cli.install.wait_ready", lambda manifest, timeout_seconds: True)+ monkeypatch.setattr(+ "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds, **kwargs: True+ )result = runner.invoke(main, ["install", "agent", "ensure"])assert result.exit_code == 0, result.output@@ -1458,6 +1996,7 @@ def test_install_agent_ensure_no_duplicate_spawn_after_lock_recheck(monkeypatch)# First probe_ready (before lock) returns False, second (after lock) returns Trueprobe_results = iter([False, True])monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: next(probe_results))monkeypatch.setattr(@@ -1557,7 +2096,9 @@ def test_install_start_reconciles_a_healthy_deployment(monkeypatch) -> None:)monkeypatch.setattr("headroom.cli.install.save_manifest", lambda m: None)monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda m: None)- monkeypatch.setattr("headroom.cli.install.wait_ready", lambda m, timeout_seconds=None: True)+ monkeypatch.setattr(+ "headroom.cli.install.wait_ready", lambda m, timeout_seconds=None, **kwargs: True+ )result = runner.invoke(main, ["install", "start"])@@ -1597,7 +2138,9 @@ def test_install_start_still_skips_when_nothing_is_pending(monkeypatch) -> None:)monkeypatch.setattr("headroom.cli.install.save_manifest", lambda m: None)monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda m: None)- monkeypatch.setattr("headroom.cli.install.wait_ready", lambda m, timeout_seconds=None: True)+ monkeypatch.setattr(+ "headroom.cli.install.wait_ready", lambda m, timeout_seconds=None, **kwargs: True+ )result = runner.invoke(main, ["install", "start"])diff --git a/tests/test_cli/test_wrap_persistent.py b/tests/test_cli/test_wrap_persistent.pyindex 92472881e..6f156308f 100644--- a/tests/test_cli/test_wrap_persistent.py+++ b/tests/test_cli/test_wrap_persistent.py@@ -318,6 +318,28 @@ def test_recover_persistent_proxy_warns_for_task_deployment(monkeypatch) -> Noneassert wrap_cli._recover_persistent_proxy(8787) is False+def test_recover_persistent_proxy_routes_docker_task_to_docker_supervisor(monkeypatch) -> None:+ calls: list[str] = []++ class DockerTaskManifest(_Manifest):+ preset = "persistent-task"+ runtime_kind = "docker"++ monkeypatch.setattr(wrap_cli, "_find_persistent_manifest", lambda port: DockerTaskManifest())+ monkeypatch.setattr("headroom.install.health.probe_ready", lambda url: False)+ monkeypatch.setattr("headroom.install.runtime.runtime_ready", lambda manifest: False)+ monkeypatch.setattr(+ "headroom.install.runtime.start_persistent_docker",+ lambda manifest: calls.append("docker"),+ )+ monkeypatch.setattr(+ "headroom.install.runtime.wait_ready", lambda manifest, timeout_seconds=45: True+ )++ assert wrap_cli._recover_persistent_proxy(8787) is True+ assert calls == ["docker"]++def test_ensure_proxy_restarts_idle_stale_ephemeral_proxy(monkeypatch) -> None:calls: list[object] = []health = {diff --git a/tests/test_install/test_paths.py b/tests/test_install/test_paths.pyindex 2ddefbdd8..8fa6be370 100644--- a/tests/test_install/test_paths.py+++ b/tests/test_install/test_paths.py@@ -22,6 +22,9 @@ def test_profile_and_artifact_paths(monkeypatch, tmp_path: Path) -> None:assert install_paths.deploy_root() == tmp_path / "deploy"assert install_paths.profile_root("demo") == tmp_path / "deploy" / "demo"assert install_paths.manifest_path("demo") == tmp_path / "deploy" / "demo" / "manifest.json"+ assert install_paths.recovery_manifest_path("demo") == (+ tmp_path / "deploy" / "demo.recovery.json"+ )assert install_paths.log_path("demo") == tmp_path / "deploy" / "demo" / "runner.log"assert install_paths.pid_path("demo") == tmp_path / "deploy" / "demo" / "runner.pid"assert (diff --git a/tests/test_install/test_planner.py b/tests/test_install/test_planner.pyindex d5169acee..dfe5f4525 100644--- a/tests/test_install/test_planner.py+++ b/tests/test_install/test_planner.py@@ -1,5 +1,7 @@from __future__ import annotations+import sys+import clickimport pytest@@ -54,6 +56,33 @@ def test_build_manifest_for_persistent_docker_sets_expected_defaults() -> None:assert "HEADROOM_KOMPRESS_BACKEND" not in manifest.base_env+def test_build_manifest_docker_runtime_never_installs_native_supervisor() -> None:+ manifest = build_manifest(+ profile="default",+ preset=InstallPreset.PERSISTENT_SERVICE.value,+ runtime_kind="docker",+ scope="user",+ provider_mode="manual",+ targets=[],+ port=8787,+ backend="anthropic",+ anyllm_provider=None,+ region=None,+ proxy_mode="token",+ memory_enabled=False,+ telemetry_enabled=False,+ image="ghcr.io/headroomlabs-ai/headroom:latest",+ )++ assert manifest.supervisor_kind == "none"+ expected_preset = (+ InstallPreset.PERSISTENT_TASK.value+ if sys.platform.startswith("win")+ else InstallPreset.PERSISTENT_SERVICE.value+ )+ assert manifest.preset == expected_preset++def test_build_manifest_python_runtime_keeps_explicit_memory_db_path() -> None:manifest = build_manifest(profile="default",diff --git a/tests/test_install/test_providers.py b/tests/test_install/test_providers.pyindex 837847150..ccbf30dcd 100644--- a/tests/test_install/test_providers.py+++ b/tests/test_install/test_providers.py@@ -432,6 +432,65 @@ def test_apply_mutations_runs_openclaw_for_user_scope(monkeypatch, tmp_path: Patassert [mutation.kind for mutation in mutations] == ["openclaw-wrap"]+def test_provider_mutation_failure_reverts_prior_returned_record(+ monkeypatch, tmp_path: Path+) -> None:+ import headroom.providers.install_registry as registry++ manifest = _manifest(tmp_path)+ manifest.targets = ["first", "second"]+ first = ManagedMutation(target="first", kind="first")+ reverted: list[ManagedMutation] = []++ def apply_first(current):+ return first++ def fail_second(current):+ raise RuntimeError("second mutation failed")++ def revert(mutation, current):+ reverted.append(mutation)++ monkeypatch.setattr(+ registry,+ "_PROVIDER_SCOPE_HANDLERS",+ {"first": (apply_first, revert), "second": (fail_second, revert)},+ )++ with pytest.raises(RuntimeError, match="second mutation failed"):+ registry.apply_provider_scope_mutations(manifest)+ assert reverted == [first]+++def test_provider_rollback_failure_preserves_unreverted_records(+ monkeypatch, tmp_path: Path+) -> None:+ import headroom.providers.install_registry as registry++ manifest = _manifest(tmp_path)+ manifest.targets = ["first", "second"]+ first = ManagedMutation(target="first", kind="first")++ def apply_first(current):+ return first++ def fail_second(current):+ raise RuntimeError("second mutation failed")++ def fail_revert(mutation, current):+ raise RuntimeError("rollback unavailable")++ monkeypatch.setattr(+ registry,+ "_PROVIDER_SCOPE_HANDLERS",+ {"first": (apply_first, fail_revert), "second": (fail_second, fail_revert)},+ )++ with pytest.raises(RuntimeError, match="rollback unavailable"):+ registry.apply_provider_scope_mutations(manifest)+ assert manifest.mutations == [first]++def test_claude_build_install_env_returns_proxy_base_url() -> None:# Arrange / Actenv = build_claude_install_env(port=5566, backend="ignored")@@ -937,3 +996,76 @@ def test_revert_opencode_provider_scope_fallback_on_oserror(monkeypatch, tmp_patassert backup_path.exists() # backup preserved when copy failsassert not config_path.exists() or "headroom" not in config_path.read_text()+++def test_apply_mutations_records_each_provider_mutation_once(monkeypatch, tmp_path: Path) -> None:+ import headroom.providers.install_registry as registry+ from headroom.install.providers import apply_mutations++ manifest = _manifest(tmp_path)+ manifest.targets = ["first"]+ first = ManagedMutation(target="first", kind="first")+ monkeypatch.setattr(+ registry,+ "_PROVIDER_SCOPE_HANDLERS",+ {"first": (lambda current: first, lambda mutation, current: None)},+ )++ assert apply_mutations(manifest) == [first]+ assert manifest.mutations == [first]+++def test_unix_env_failure_restores_previous_shell_contents(monkeypatch, tmp_path: Path) -> None:+ from headroom import fsutil+ from headroom.install import providers++ manifest = _manifest(tmp_path)+ manifest.scope = "user"+ bashrc = tmp_path / ".bashrc"+ zshrc = tmp_path / ".zshrc"+ previous = (+ '# >>> headroom persistent env >>>\nexport OLD="1"\n# <<< headroom persistent env <<<\n'+ )+ bashrc.write_text(previous, encoding="utf-8")+ monkeypatch.setattr(providers, "unix_user_env_targets", lambda: [bashrc, zshrc])+ real_write = fsutil.write_text++ def write_text(path, content):+ if Path(path) == zshrc:+ raise OSError("disk full")+ real_write(path, content)++ monkeypatch.setattr(fsutil, "write_text", write_text)++ with pytest.raises(OSError, match="disk full"):+ providers._apply_unix_env_scope(manifest)+ assert bashrc.read_text(encoding="utf-8") == previous+ assert not zshrc.exists()+++def test_unix_env_failure_restores_earlier_targets_when_existing_target_rejects_writes(+ monkeypatch, tmp_path: Path+) -> None:+ from headroom import fsutil+ from headroom.install import providers++ manifest = _manifest(tmp_path)+ manifest.scope = "user"+ bashrc = tmp_path / ".bashrc"+ zshrc = tmp_path / ".zshrc"+ bashrc.write_text("# bash\n", encoding="utf-8")+ zshrc.write_text("# zsh\n", encoding="utf-8")+ monkeypatch.setattr(providers, "unix_user_env_targets", lambda: [bashrc, zshrc])+ real_write = fsutil.write_text++ def write_text(path, content):+ if Path(path) == zshrc:+ raise OSError("read-only")+ real_write(path, content)++ monkeypatch.setattr(fsutil, "write_text", write_text)++ with pytest.raises(OSError, match="read-only"):+ providers._apply_unix_env_scope(manifest)+ assert bashrc.read_text(encoding="utf-8") == "# bash\n"+ assert zshrc.read_text(encoding="utf-8") == "# zsh\n"diff --git a/tests/test_install/test_runtime.py b/tests/test_install/test_runtime.pyindex 046b6a090..183e7a21c 100644--- a/tests/test_install/test_runtime.py+++ b/tests/test_install/test_runtime.py@@ -14,6 +14,8 @@ from headroom.install.runtime import (_clear_pid,_deployment_env,_mount_source,+ _process_identity,+ _process_matches_runtime,_read_pid,_runtime_env,_write_pid,@@ -21,6 +23,7 @@ from headroom.install.runtime import (build_runtime_command,resolve_headroom_command,run_foreground,+ runtime_ownership,runtime_status,start_detached_agent,start_persistent_docker,@@ -187,7 +190,7 @@ def test_build_runtime_command_preserves_non_ascii_ambient_token(captured: dict[str, list[str] | dict[str, str]] = {}- def fake_run(command: list[str], **kwargs: object) -> None:+ def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess:captured["command"] = commandcontainer_env: dict[str, str] = {}for index, arg in enumerate(command):@@ -200,6 +203,7 @@ def test_build_runtime_command_preserves_non_ascii_ambient_token(else:container_env[value] = os.environ[value]captured["container_env"] = container_env+ return subprocess.CompletedProcess(command, 0)monkeypatch.setattr("headroom.install.runtime.subprocess.run", fake_run)start_persistent_docker(manifest)@@ -548,8 +552,9 @@ def test_runtime_start_lock_blocks_another_process(monkeypatch, tmp_path: Path)assert result.stdout.strip() == "False"-def test_run_foreground_and_detached_helpers(monkeypatch, tmp_path: Path) -> None:+def test_non_launchd_foreground_and_detached_helpers(monkeypatch, tmp_path: Path) -> None:monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "win32")monkeypatch.setattr("headroom.install.runtime.build_runtime_command", lambda manifest: ["headroom", "proxy"])@@ -631,6 +636,410 @@ def test_run_foreground_and_detached_helpers(monkeypatch, tmp_path: Path) -> Nonassert start_detached_agent("demo") is fake_proc_posix+def test_launchd_exec_handoff_uses_current_pid_and_never_popens(+ monkeypatch, tmp_path: Path+) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")+ monkeypatch.setattr(+ "headroom.install.runtime.build_runtime_command",+ lambda manifest: ["/usr/bin/python3", "-m", "headroom.cli", "proxy"],+ )+ expected_env = {"HEADROOM_TEST": "exec"}+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: expected_env)+ monkeypatch.setattr("headroom.install.runtime.os.getpid", lambda: 4321)++ exec_calls: list[tuple[str, list[str], dict[str, str]]] = []+ pid_at_exec: list[int | None] = []++ def fake_execvpe(file: str, args: list[str], env: dict[str, str]) -> None:+ exec_calls.append((file, args, env))+ pid_at_exec.append(_read_pid("default"))+ raise OSError("exec failed")++ monkeypatch.setattr("headroom.install.runtime.os.execvpe", fake_execvpe)+ monkeypatch.setattr("headroom.install.runtime.os.dup2", lambda source, target: None)+ monkeypatch.setattr(+ "headroom.install.runtime.subprocess.Popen",+ lambda *args, **kwargs: pytest.fail("launchd handoff must not spawn a child"),+ )++ with pytest.raises(OSError, match="exec failed"):+ run_foreground(_python_service_manifest())++ assert exec_calls == [+ (+ "/usr/bin/python3",+ ["/usr/bin/python3", "-m", "headroom.cli", "proxy"],+ expected_env,+ )+ ]+ assert pid_at_exec == [4321]+ assert _read_pid("default") is None+++def test_launchd_exec_preserves_runtime_contract(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")+ command = ["/usr/bin/python3", "-m", "headroom.cli", "proxy", "--port", "8787"]+ env = {"HEADROOM_DEPLOYMENT_PROFILE": "default", "HEADROOM_SECRET": "test"}+ monkeypatch.setattr("headroom.install.runtime.build_runtime_command", lambda manifest: command)+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: env)+ monkeypatch.setattr("headroom.install.runtime.os.getpid", lambda: 9876)++ dup2_calls: list[tuple[int, int]] = []+ exec_calls: list[tuple[str, list[str], dict[str, str]]] = []++ class FakeLog:+ def __enter__(self):+ return self++ def __exit__(self, exc_type, exc_value, traceback) -> None:+ return None++ def fileno(self) -> int:+ return 42++ monkeypatch.setattr(+ "headroom.install.runtime.open", lambda *args, **kwargs: FakeLog(), raising=False+ )++ def fake_dup2(source: int, target: int) -> None:+ dup2_calls.append((source, target))++ def fake_execvpe(file: str, args: list[str], runtime_env: dict[str, str]) -> None:+ exec_calls.append((file, args, runtime_env))+ raise OSError("exec failed")++ monkeypatch.setattr("headroom.install.runtime.os.dup2", fake_dup2)+ monkeypatch.setattr("headroom.install.runtime.os.execvpe", fake_execvpe)+ monkeypatch.setattr(+ "headroom.install.runtime.subprocess.Popen",+ lambda *args, **kwargs: pytest.fail("launchd handoff must not spawn a child"),+ )++ with pytest.raises(OSError, match="exec failed"):+ run_foreground(_python_service_manifest())++ assert exec_calls == [(command[0], command, env)]+ assert len(dup2_calls) == 2+ assert dup2_calls[0][1:] == (1,)+ assert dup2_calls[1][1:] == (2,)+ assert dup2_calls == [(42, 1), (42, 2)]+++def test_docker_runtime_rejects_run_foreground_split_owner(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")+ command = ["docker", "run", "--rm", "headroom:test"]+ env = {"HEADROOM_DEPLOYMENT_RUNTIME": "docker"}+ monkeypatch.setattr("headroom.install.runtime.build_runtime_command", lambda manifest: command)+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: env)++ monkeypatch.setattr(+ "headroom.install.runtime.subprocess.Popen",+ lambda *args, **kwargs: pytest.fail("Docker supervisor must own the container lifecycle"),+ )+ monkeypatch.setattr(+ "headroom.install.runtime.os.execvpe",+ lambda *args: pytest.fail("Docker launchd service must keep the daemon-owned Popen path"),+ )++ manifest = _python_service_manifest()+ manifest.runtime_kind = "docker"+ with pytest.raises(RuntimeError, match="Docker deployments must be started"):+ run_foreground(manifest)+++def test_runtime_ownership_classifier_covers_launchd_docker_and_foreground(monkeypatch) -> None:+ python_manifest = _python_service_manifest()+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")+ assert runtime_ownership(python_manifest) == "launchd-exec"++ for preset, supervisor in (+ (InstallPreset.PERSISTENT_DOCKER.value, "none"),+ (InstallPreset.PERSISTENT_SERVICE.value, "service"),+ (InstallPreset.PERSISTENT_TASK.value, "task"),+ ):+ docker_manifest = _python_service_manifest()+ docker_manifest.preset = preset+ docker_manifest.runtime_kind = "docker"+ docker_manifest.supervisor_kind = supervisor+ assert runtime_ownership(docker_manifest) == "docker-supervisor"++ python_manifest.preset = "persistent-task"+ python_manifest.supervisor_kind = "task"+ assert runtime_ownership(python_manifest) == "popen"+++def test_launchd_transition_keeps_one_listener_owner(monkeypatch, tmp_path: Path) -> None:+ """A launchd restart replaces the listener in place instead of orphaning a child."""+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")+ monkeypatch.setattr(+ "headroom.install.runtime.build_runtime_command",+ lambda manifest: ["/usr/bin/python3", "-m", "headroom.cli", "proxy"],+ )+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: {})+ monkeypatch.setattr("headroom.install.runtime.os.getpid", lambda: 4321)+ monkeypatch.setattr("headroom.install.runtime.os.dup2", lambda source, target: None)+ monkeypatch.setattr("headroom.install.runtime.subprocess.Popen", lambda *a, **k: pytest.fail())++ exec_pids: list[int | None] = []++ class ExecHandoff(Exception):+ pass++ monkeypatch.setattr(+ "headroom.install.runtime.os.execvpe",+ lambda file, args, env: (exec_pids.append(_read_pid("default")), raise_exec()),+ )+ monkeypatch.setattr("headroom.install.runtime._clear_pid", lambda *args, **kwargs: None)++ def raise_exec() -> None:+ raise ExecHandoff++ manifest = _python_service_manifest()+ assert runtime_ownership(manifest) == "launchd-exec"+ with pytest.raises(ExecHandoff):+ run_foreground(manifest)+ with pytest.raises(ExecHandoff):+ run_foreground(manifest)++ assert exec_pids == [4321, 4321]+ assert _read_pid("default") == 4321+++def test_status_and_stop_reject_a_reused_pid_without_signaling(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _python_service_manifest()+ _write_pid("default", 123)+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)+ monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: False)+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, current: False)+ monkeypatch.setattr(+ "headroom.install.runtime.os.kill",+ lambda *args: pytest.fail("reused PID must not be signaled"),+ )+ assert runtime_status(manifest) == "stopped"+ _write_pid("default", 123)+ stop_runtime(manifest)+ assert _read_pid("default") is None+++def test_process_identity_must_match_headroom_command(monkeypatch) -> None:+ class FakeProcess:+ def __init__(self, pid: int) -> None:+ self.pid = pid++ def cmdline(self) -> list[str]:+ return ["python", "unrelated-service.py", "--port", "8787"]++ def environ(self) -> dict[str, str]:+ return {+ "HEADROOM_DEPLOYMENT_PROFILE": "default",+ "HEADROOM_DEPLOYMENT_RUNTIME": "python",+ }++ monkeypatch.setitem(sys.modules, "psutil", types.SimpleNamespace(Process=FakeProcess))+ assert not _process_matches_runtime(4321, _python_service_manifest())++ class MatchingProcess(FakeProcess):+ def cmdline(self) -> list[str]:+ return ["python", "-m", "headroom.cli", "proxy", "--port", "8787"]++ def environ(self) -> dict[str, str]:+ return {+ "HEADROOM_DEPLOYMENT_PROFILE": "default",+ "HEADROOM_DEPLOYMENT_RUNTIME": "python",+ }++ monkeypatch.setitem(sys.modules, "psutil", types.SimpleNamespace(Process=MatchingProcess))+ assert _process_matches_runtime(4321, _python_service_manifest())+++def test_process_identity_falls_back_without_psutil(monkeypatch) -> None:+ manifest = _python_service_manifest()+ monkeypatch.setattr(+ "headroom.install.runtime._read_proc_metadata",+ lambda pid: (+ ["python", "-m", "headroom.cli", "proxy", "--port", "8787"],+ {+ "HEADROOM_DEPLOYMENT_PROFILE": "default",+ "HEADROOM_DEPLOYMENT_RUNTIME": "python",+ },+ ),+ )+ monkeypatch.setitem(sys.modules, "psutil", None)+ assert _process_matches_runtime(4321, manifest)+++def test_process_identity_uses_macos_ps_fallback_without_psutil(monkeypatch) -> None:+ manifest = _python_service_manifest()+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")+ monkeypatch.setitem(sys.modules, "psutil", None)++ class Result:+ returncode = 0+ stdout = (+ "/usr/bin/python3 -m headroom.cli proxy --port 8787 "+ "HEADROOM_DEPLOYMENT_PROFILE=default "+ "HEADROOM_DEPLOYMENT_RUNTIME=python"+ )++ monkeypatch.setattr("headroom.install.runtime.run", lambda *a, **k: Result())+ assert _process_matches_runtime(4321, manifest)+++def test_process_identity_uses_windows_wmi_command_line_without_psutil(monkeypatch) -> None:+ manifest = _python_service_manifest()+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "win32")+ monkeypatch.setitem(sys.modules, "psutil", None)++ class Result:+ returncode = 0+ stdout = (+ '"C:\\Python\\python.exe" -m headroom.cli proxy --port 8787 '+ "--headroom-deployment-profile default --headroom-deployment-runtime python"+ )++ calls: list[list[str]] = []+ monkeypatch.setattr(+ "headroom.install.runtime.run",+ lambda command, **kwargs: calls.append(command) or Result(),+ )++ assert _process_matches_runtime(4321, manifest)+ assert calls[0][:3] == ["powershell.exe", "-NoProfile", "-NonInteractive"]+++def test_windows_runtime_status_and_stop_use_wmi_identity(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "win32")+ monkeypatch.setitem(sys.modules, "psutil", None)+ manifest = _python_service_manifest()+ _write_pid("default", 4321)+ pid_states = iter([True, True, False, False])+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: next(pid_states))++ class Result:+ returncode = 0+ stdout = (+ '"C:\\Python\\python.exe" -m headroom.cli proxy --port 8787 '+ "--headroom-deployment-profile default --headroom-deployment-runtime python"+ )++ monkeypatch.setattr("headroom.install.runtime.run", lambda *a, **k: Result())+ killed: list[tuple[int, int]] = []+ monkeypatch.setattr(+ "headroom.install.runtime.os.kill", lambda pid, sig: killed.append((pid, sig))+ )++ assert runtime_status(manifest) == "running"+ stop_runtime(manifest)+ assert killed == [(4321, signal.SIGTERM)]+ assert _read_pid("default") is None+++def test_windows_wmi_same_port_without_deployment_markers_is_unknown_and_safe(+ monkeypatch, tmp_path: Path+) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "win32")+ monkeypatch.setitem(sys.modules, "psutil", None)+ manifest = _python_service_manifest()++ class Result:+ returncode = 0+ stdout = '"C:\\Python\\python.exe" -m headroom.cli proxy --port 8787'++ monkeypatch.setattr("headroom.install.runtime.run", lambda *a, **k: Result())+ _write_pid("default", 4321)+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)+ monkeypatch.setattr(+ "headroom.install.runtime.os.kill",+ lambda *args: pytest.fail("unidentified same-port process must not be signaled"),+ )++ assert _process_identity(4321, manifest) is None+ assert runtime_status(manifest) == "unknown"+ with pytest.raises(RuntimeError, match="identity unknown"):+ stop_runtime(manifest)+ assert _read_pid("default") == 4321+++def test_process_identity_without_supported_metadata_is_unknown_and_safe(+ monkeypatch, tmp_path: Path+) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setitem(sys.modules, "psutil", None)+ monkeypatch.setattr("headroom.install.runtime._read_proc_metadata", lambda pid: None)+ manifest = _python_service_manifest()+ _write_pid("default", 4321)+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)+ monkeypatch.setattr(+ "headroom.install.runtime.os.kill",+ lambda *args: pytest.fail("unknown identity must not be signaled"),+ )++ assert _process_identity(4321, manifest) is None+ assert runtime_status(manifest) == "unknown"+ with pytest.raises(RuntimeError, match="identity unknown"):+ stop_runtime(manifest)+ assert _read_pid("default") == 4321+++def test_runtime_status_does_not_use_ready_probe_for_failed_identity(+ monkeypatch, tmp_path: Path+) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _python_service_manifest()+ _write_pid("default", 4321)+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, m: False)+ monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: True)++ assert runtime_status(manifest) == "stopped"+ assert _read_pid("default") is None+++def test_process_identity_rejects_current_pid(monkeypatch) -> None:+ monkeypatch.setattr("headroom.install.runtime.os.getpid", lambda: 4321)+ assert not _process_matches_runtime(4321, _python_service_manifest())+++def test_darwin_task_preserves_popen_path(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")+ monkeypatch.setattr(+ "headroom.install.runtime.build_runtime_command", lambda manifest: ["headroom", "proxy"]+ )+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: {"ENV": "1"})++ class FakeProc:+ pid = 8765++ def wait(self, timeout: int | None = None) -> int:+ return 0++ popen_calls: list[tuple[list[str], dict]] = []+ monkeypatch.setattr(+ "headroom.install.runtime.subprocess.Popen",+ lambda command, **kwargs: popen_calls.append((command, kwargs)) or FakeProc(),+ )+ monkeypatch.setattr(+ "headroom.install.runtime.os.execvpe",+ lambda *args: pytest.fail("Darwin task must preserve Popen"),+ )++ task_manifest = _python_service_manifest()+ task_manifest.preset = "persistent-task"+ task_manifest.supervisor_kind = "task"+ assert run_foreground(task_manifest) == 0+ assert popen_calls[0][0] == ["headroom", "proxy"]+ assert _read_pid("default") is None++def test_start_detached_agent_closes_parent_log_fd(monkeypatch, tmp_path: Path) -> None:"""The parent must close its copy of the log file after Popen.@@ -685,7 +1094,10 @@ def test_start_stop_wait_and_runtime_status_branches(monkeypatch, tmp_path: Pathcalls: list[list[str]] = []monkeypatch.setattr("headroom.install.runtime.subprocess.run",- lambda command, **kwargs: calls.append(command) or type("Result", (), {"stdout": ""})(),+ lambda command, **kwargs: (+ calls.append(command)+ or type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()+ ),)monkeypatch.setattr("headroom.install.runtime.build_runtime_command",@@ -749,6 +1161,9 @@ def test_start_stop_wait_and_runtime_status_branches(monkeypatch, tmp_path: Pathmonkeypatch.setattr("headroom.install.runtime.os.kill", lambda pid, sig: killed.append((pid, sig)))+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, manifest: True)+ pid_states = iter([True, False, False, True, False, False])+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: next(pid_states))stop_runtime(python_manifest)assert killed == [(123, signal.SIGTERM)]assert _read_pid("default") is None@@ -789,6 +1204,7 @@ def test_start_stop_wait_and_runtime_status_branches(monkeypatch, tmp_path: Path_write_pid("default", 125)monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: False)assert runtime_status(python_manifest) == "stopped"+ assert _read_pid("default") is Nonedef test_stop_runtime_for_docker_stops_and_removes_container(monkeypatch) -> None:@@ -809,7 +1225,10 @@ def test_stop_runtime_for_docker_stops_and_removes_container(monkeypatch) -> Nonmonkeypatch.setattr("headroom.install.runtime.subprocess.run",- lambda command, **kwargs: calls.append(command),+ lambda command, **kwargs: (+ calls.append(command)+ or type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()+ ),)stop_runtime(manifest)@@ -820,6 +1239,101 @@ def test_stop_runtime_for_docker_stops_and_removes_container(monkeypatch) -> Non]+def test_stop_runtime_rejects_unknown_identity(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _python_service_manifest()+ _write_pid("default", 4321)+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, current: None)+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)++ with pytest.raises(RuntimeError, match="identity unknown"):+ stop_runtime(manifest)+ assert _read_pid("default") == 4321+++def test_stop_runtime_rejects_malformed_pid_file(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _python_service_manifest()+ pid_file = tmp_path / ".headroom" / "deploy" / "default" / "runner.pid"+ pid_file.parent.mkdir(parents=True)+ pid_file.write_text("not-a-pid", encoding="utf-8")++ with pytest.raises(RuntimeError, match="runner.pid is invalid"):+ stop_runtime(manifest)+ assert pid_file.exists()+++def test_runtime_status_reports_unknown_for_malformed_pid_file(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _python_service_manifest()+ pid_file = tmp_path / ".headroom" / "deploy" / "default" / "runner.pid"+ pid_file.parent.mkdir(parents=True)+ pid_file.write_text("not-a-pid", encoding="utf-8")++ assert runtime_status(manifest) == "unknown"+ assert pid_file.exists()+++def test_stop_runtime_rejects_identity_change_before_clearing_pid(+ monkeypatch, tmp_path: Path+) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _python_service_manifest()+ _write_pid("default", 4321)+ identities = iter([True, False])+ monkeypatch.setattr(+ "headroom.install.runtime._process_identity", lambda pid, current: next(identities)+ )+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)+ monkeypatch.setattr("headroom.install.runtime.os.kill", lambda pid, sig: None)++ with pytest.raises(RuntimeError, match="identity changed"):+ stop_runtime(manifest)+ assert _read_pid("default") == 4321+++def test_stop_runtime_keeps_pid_when_sigterm_does_not_stop_process(+ monkeypatch, tmp_path: Path+) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _python_service_manifest()+ _write_pid("default", 4321)+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, current: True)+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)+ monkeypatch.setattr("headroom.install.runtime.os.kill", lambda pid, sig: None)+ monkeypatch.setattr("headroom.install.runtime.time.sleep", lambda seconds: None)++ with pytest.raises(RuntimeError, match="remained alive"):+ stop_runtime(manifest)+ assert _read_pid("default") == 4321+++def test_stop_runtime_surfaces_docker_stop_failure(monkeypatch) -> None:+ manifest = DeploymentManifest(+ profile="default",+ preset="persistent-docker",+ runtime_kind="docker",+ supervisor_kind="none",+ scope="user",+ provider_mode="manual",+ targets=[],+ port=8787,+ host="127.0.0.1",+ backend="anthropic",+ container_name="headroom-default",+ )++ class Result:+ returncode = 1+ stdout = ""+ stderr = "permission denied"++ monkeypatch.setattr("headroom.install.runtime.run", lambda *args, **kwargs: Result())++ with pytest.raises(RuntimeError, match="docker stop failed"):+ stop_runtime(manifest)++def test_runtime_status_reads_container_and_pid_state(monkeypatch, tmp_path: Path) -> None:docker_manifest = DeploymentManifest(profile="default",@@ -850,6 +1364,7 @@ def test_runtime_status_reads_container_and_pid_state(monkeypatch, tmp_path: Patpid_file.parent.mkdir(parents=True)pid_file.write_text("123", encoding="utf-8")monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, manifest: True)python_manifest = DeploymentManifest(profile="default",preset="persistent-service",@@ -892,6 +1407,7 @@ def test_runtime_status_reports_live_pid_without_terminating(monkeypatch, tmp_pamonkeypatch.setattr("headroom.install.runtime.os.kill", fail_kill)monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, manifest: True)assert runtime_status(_python_service_manifest()) == "running"assert pid_file.exists() # status left the deployment untouched@@ -960,6 +1476,17 @@ class TestRestartCurrentDeployment:assert result["command"] == "headroom install restart --profile default"assert popen_calls == [] # cannot run docker from inside the container+ def test_runtime_env_docker_returns_host_command_without_restarting(self, monkeypatch) -> None:+ from headroom.install import runtime as rt++ monkeypatch.setenv("HEADROOM_DEPLOYMENT_PROFILE", "default")+ monkeypatch.setenv("HEADROOM_DEPLOYMENT_RUNTIME", "docker")+ monkeypatch.delenv("HEADROOM_DEPLOYMENT_PRESET", raising=False)+ monkeypatch.setattr(rt, "load_manifest", lambda profile: None)++ _manifest, mode = rt.detect_current_deployment()+ assert mode == "docker"+def test_task_mode_detected_and_not_restarted(self, monkeypatch) -> None:"""A persistent-task deployment must not be told it's a self-restartable 'service'.@@ -1010,3 +1537,17 @@ class TestRestartCurrentDeployment:assert result["restarted"] is Trueassert result["mode"] == "service"assert recorded["command"][-4:] == ["install", "restart", "--profile", "default"]+++def test_stop_runtime_clears_pid_of_dead_process(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _python_service_manifest()+ _write_pid("default", 4321)+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: False)+ monkeypatch.setattr(+ "headroom.install.runtime._process_identity",+ lambda pid, current: pytest.fail("a dead PID has no identity to check"),+ )++ stop_runtime(manifest)+ assert _read_pid("default") is Nonediff --git a/tests/test_install/test_state.py b/tests/test_install/test_state.pyindex 4622e3f04..647cfd37f 100644--- a/tests/test_install/test_state.py+++ b/tests/test_install/test_state.py@@ -9,9 +9,11 @@ from headroom.install.models import ArtifactRecord, DeploymentManifest, ManagedMfrom headroom.install.state import (ManifestError,delete_manifest,+ delete_recovery_manifest,list_manifests,load_manifest,save_manifest,+ save_recovery_manifest,)@@ -72,6 +74,22 @@ def test_save_manifest_writes_atomically(monkeypatch, tmp_path: Path) -> None:assert load_manifest("default") is not None+def test_recovery_manifest_preserves_mutations_and_artifacts(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ manifest = _manifest()++ save_recovery_manifest(manifest)++ recovery = tmp_path / ".headroom" / "deploy" / "default.recovery.json"+ assert recovery.exists()+ payload = json.loads(recovery.read_text(encoding="utf-8"))+ assert payload["mutations"][0]["kind"] == "shell-block"+ assert payload["artifacts"][0]["kind"] == "script"+ assert load_manifest("default") is None+ delete_recovery_manifest("default")+ assert not recovery.exists()++def test_list_manifests_ignores_invalid_payloads(monkeypatch, tmp_path: Path) -> None:monkeypatch.setattr(Path, "home", lambda: tmp_path)valid = _manifest()@@ -154,3 +172,34 @@ def test_delete_manifest_removes_profile_root(monkeypatch, tmp_path: Path) -> Noassert load_manifest("default") is Noneassert not extra_file.parent.exists()+++def test_delete_manifest_propagates_removal_failure(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ root = tmp_path / ".headroom" / "deploy" / "default"+ root.mkdir(parents=True)+ monkeypatch.setattr(+ "headroom.install.state.shutil.rmtree", lambda path: (_ for _ in ()).throw(OSError("busy"))+ )++ with pytest.raises(OSError, match="busy"):+ delete_manifest("default")++[email protected]_newline+def test_manifest_writes_pin_lf(monkeypatch, tmp_path: Path) -> None:+ import headroom.install.state as state++ monkeypatch.setattr(Path, "home", lambda: tmp_path)+ calls: list[dict] = []+ real_fdopen = state.os.fdopen++ def fdopen(fd, *args, **kwargs):+ calls.append(kwargs)+ return real_fdopen(fd, *args, **kwargs)++ monkeypatch.setattr(state.os, "fdopen", fdopen)+ save_manifest(_manifest())+ save_recovery_manifest(_manifest())++ assert [call.get("newline") for call in calls] == ["\n", "\n"]diff --git a/tests/test_install/test_supervisors.py b/tests/test_install/test_supervisors.pyindex 400b1a648..f3310bc0f 100644--- a/tests/test_install/test_supervisors.py+++ b/tests/test_install/test_supervisors.py@@ -291,6 +291,8 @@ def test_macos_launchd_plist_switches_between_keepalive_and_interval(assert service_path == tmp_path / "Library" / "LaunchAgents" / "com.headroom.default.plist"assert "<key>KeepAlive</key>" in service_contentassert "<key>StartInterval</key>" not in service_content+ assert "<key>ProgramArguments</key>" in service_content+ assert "run-headroom.sh" in service_contenttask_manifest = _manifest(profile="tasky", supervisor=SupervisorKind.TASK.value)task_path, task_content = _macos_launchd_plist(@@ -795,6 +797,32 @@ def test_install_supervisor_darwin_windows_and_unsupported(monkeypatch, tmp_pathinstall_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))+def test_install_supervisor_darwin_prepare_does_not_bootstrap(monkeypatch, tmp_path: Path) -> None:+ run_script = tmp_path / "run-headroom.sh"+ monkeypatch.setattr(+ "headroom.install.supervisors.render_runner_scripts",+ lambda manifest: [type("Record", (), {"kind": "script", "path": str(run_script)})()],+ )+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")+ monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 123, raising=False)+ monkeypatch.setattr(+ "headroom.install.supervisors._macos_launchd_plist",+ lambda manifest, script, interval=None: (tmp_path / "job.plist", "plist"),+ )+ bootstraps: list[list[str]] = []+ monkeypatch.setattr(+ "headroom.install.supervisors._bootstrap_with_retry",+ lambda domain, path, **kwargs: bootstraps.append([domain, str(path)]),+ )+ monkeypatch.setattr(+ "headroom.install.supervisors.run", lambda *args, **kwargs: _LaunchctlResult(0)+ )++ install_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value), start=False)++ assert bootstraps == []++def test_install_supervisor_retries_bootstrap_until_launchd_settles(monkeypatch, tmp_path: Path) -> None:@@ -871,6 +899,16 @@ class _LaunchctlResult:self.stdout = stdout+def test_linux_persistent_task_stop_does_not_call_systemctl(monkeypatch) -> None:+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "linux")+ monkeypatch.setattr(+ "headroom.install.supervisors.subprocess.run",+ lambda *args, **kwargs: pytest.fail("cron task stop must not call systemctl"),+ )++ stop_supervisor(_manifest(supervisor=SupervisorKind.TASK.value))++def test_start_and_stop_supervisor_darwin_windows_and_none(monkeypatch) -> None:calls: list[list[str]] = []monkeypatch.setattr(@@ -1041,7 +1079,11 @@ def test_remove_supervisor_linux_service_cron_path_and_missing_crontab(def fake_run(command: list[str], **kwargs):calls.append(command)- return type("Result", (), {"returncode": 1, "stdout": ""})()+ if command == ["crontab", "-l"]:+ return type(+ "Result", (), {"returncode": 1, "stdout": "", "stderr": "no crontab for user"}+ )()+ return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()monkeypatch.setattr("headroom.install.supervisors.subprocess.run", fake_run)unit_path = tmp_path / "headroom-default.service"@@ -1072,11 +1114,35 @@ def test_remove_supervisor_linux_service_cron_path_and_missing_crontab(assert calls[-1] == ["crontab", "-l"]+def test_remove_supervisor_surfaces_linux_command_failure(monkeypatch, tmp_path: Path) -> None:+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "linux")++ class Result:+ returncode = 1+ stdout = ""+ stderr = "permission denied"++ monkeypatch.setattr("headroom.install.supervisors.run", lambda *args, **kwargs: Result())+ unit_path = tmp_path / "headroom-default.service"+ unit_path.write_text("unit", encoding="utf-8")+ monkeypatch.setattr(+ "headroom.install.supervisors._linux_service_unit",+ lambda manifest, script: (unit_path, "unit"),+ )++ with pytest.raises(click.ClickException, match="systemctl disable --now failed"):+ remove_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))+ assert unit_path.exists()++def test_remove_supervisor_darwin_and_windows(monkeypatch, tmp_path: Path) -> None:calls: list[list[str]] = []monkeypatch.setattr("headroom.install.supervisors.subprocess.run",- lambda command, **kwargs: calls.append(command),+ lambda command, **kwargs: (+ calls.append(command)+ or type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()+ ),)monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 55, raising=False)