Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(install): replace the launchd wrapper with the proxy listener (#3224)

Rod Boev44db755ee6d8
diff --git a/headroom/cli/init.py b/headroom/cli/init.py
index 048c5e826..26178d951 100644
--- a/headroom/cli/init.py
+++ b/headroom/cli/init.py
@@ -45,6 +45,7 @@ from headroom.install.providers import _apply_unix_env_scope, _apply_windows_env
from headroom.install.runtime import (
acquire_runtime_start_lock,
resolve_headroom_command,
+ runtime_ownership,
runtime_status,
start_detached_agent,
start_persistent_docker,
@@ -62,6 +63,16 @@ from .main import main
logger = logging.getLogger(__name__)
+
+def _wait_for_runtime_ready(manifest: Any, timeout_seconds: int) -> bool:
+ """Keep hook recovery gated by both readiness and runtime identity."""
+ try:
+ return wait_ready(manifest, timeout_seconds=timeout_seconds, require_identity=True)
+ except TypeError:
+ # Compatibility for test doubles that predate the keyword-only guard.
+ return wait_ready(manifest, timeout_seconds=timeout_seconds)
+
+
_VERBOSE_HANDLER_ATTR = "_headroom_init_verbose_handler"
_GLOBAL_PROFILE = "init-user"
@@ -784,25 +795,25 @@ def _ensure_profile_running(profile: str) -> None:
if manifest is None:
return
with _suppress_hook_output():
- if wait_ready(manifest, timeout_seconds=1):
+ if _wait_for_runtime_ready(manifest, timeout_seconds=1):
return
try:
with acquire_runtime_start_lock(manifest.profile) as acquired:
if not acquired:
return
- if wait_ready(manifest, timeout_seconds=1):
+ if _wait_for_runtime_ready(manifest, timeout_seconds=1):
return
if runtime_status(manifest) == "running":
- if wait_ready(manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS):
+ if _wait_for_runtime_ready(manifest, _STARTUP_READY_TIMEOUT_SECONDS):
return
stop_runtime(manifest)
- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:
+ if runtime_ownership(manifest) == "docker-supervisor":
start_persistent_docker(manifest)
elif manifest.supervisor_kind == SupervisorKind.SERVICE.value:
start_supervisor(manifest)
else:
start_detached_agent(manifest.profile)
- wait_ready(manifest, timeout_seconds=45)
+ _wait_for_runtime_ready(manifest, 45)
except Exception:
return
diff --git a/headroom/cli/install.py b/headroom/cli/install.py
index f5a7141c3..7ef8b6b32 100644
--- a/headroom/cli/install.py
+++ b/headroom/cli/install.py
@@ -23,11 +23,13 @@ from headroom.install.models import (
RuntimeKind,
SupervisorKind,
)
+from headroom.install.paths import recovery_manifest_path
from headroom.install.planner import build_manifest, build_tool_envs
from headroom.install.providers import apply_mutations, revert_mutations
from headroom.install.runtime import (
acquire_runtime_start_lock,
run_foreground,
+ runtime_ownership,
runtime_status,
start_detached_agent,
start_persistent_docker,
@@ -38,9 +40,12 @@ from headroom.install.runtime import (
from headroom.install.state import (
ManifestError,
delete_manifest,
+ delete_recovery_manifest,
list_manifests,
load_manifest,
save_manifest,
+ save_manifest_strict,
+ save_recovery_manifest,
)
from headroom.install.supervisors import (
install_supervisor,
@@ -155,19 +160,27 @@ def _start_deployment(manifest: DeploymentManifest, *, assume_start_lock: bool =
_start_deployment(manifest, assume_start_lock=True)
return
- if probe_ready(manifest.health_url):
- return
- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value and shutil.which("docker") is None:
+ docker_owned = runtime_ownership(manifest) == "docker-supervisor"
+ if docker_owned and shutil.which("docker") is None:
raise click.ClickException(
"Docker is required for this deployment but 'docker' was not found on PATH."
)
- if runtime_status(manifest) == "running":
- if wait_ready(manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS):
+ status = runtime_status(manifest)
+ if status == "running" and probe_ready(manifest.health_url):
+ return
+ if status == "unknown":
+ raise click.ClickException(
+ f"Cannot start deployment '{manifest.profile}': runtime identity is unavailable."
+ )
+ if status == "running":
+ if wait_ready(
+ manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS, require_identity=True
+ ):
return
stop_runtime(manifest)
try:
- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:
+ if runtime_ownership(manifest) == "docker-supervisor":
start_persistent_docker(manifest)
elif manifest.supervisor_kind == SupervisorKind.SERVICE.value:
start_supervisor(manifest)
@@ -182,25 +195,42 @@ def _start_deployment(manifest: DeploymentManifest, *, assume_start_lock: bool =
f"({' '.join(map(str, e.cmd)) if isinstance(e.cmd, list | tuple) else e.cmd})"
) from None
- if not wait_ready(manifest, timeout_seconds=45):
+ if not wait_ready(manifest, timeout_seconds=45, require_identity=True):
raise click.ClickException(
f"Deployment '{manifest.profile}' did not become ready after start."
)
def _stop_deployment(manifest: DeploymentManifest) -> None:
- if manifest.supervisor_kind == SupervisorKind.SERVICE.value:
- stop_supervisor(manifest)
- stop_runtime(manifest)
- # Stopping returns before the old process has finished shutting down, so it
- # can keep answering /readyz. `_start_deployment` treats a ready endpoint as
- # "already running" and would skip the start, leaving the deployment stopped
- # once the old process exits. Block until it is really gone.
- if not wait_stopped(manifest):
- raise click.ClickException(
- f"Deployment '{manifest.profile}' is still answering on "
- f"{manifest.health_url} after stop."
- )
+ errors: list[tuple[str, Exception]] = []
+ if manifest.supervisor_kind in {
+ SupervisorKind.SERVICE.value,
+ SupervisorKind.TASK.value,
+ }:
+ try:
+ stop_supervisor(manifest)
+ except Exception as exc:
+ errors.append(("supervisor stop", exc))
+ try:
+ stop_runtime(manifest)
+ except Exception as exc:
+ errors.append(("runtime stop", exc))
+ try:
+ if not wait_stopped(manifest):
+ timeout_error = click.ClickException(
+ f"Deployment '{manifest.profile}' is still answering on "
+ f"{manifest.health_url} after stop."
+ )
+ if errors:
+ errors.append(("runtime shutdown", timeout_error))
+ else:
+ raise timeout_error
+ except click.ClickException:
+ raise
+ except Exception as exc:
+ errors.append(("runtime shutdown", exc))
+ if errors:
+ raise RuntimeError("; ".join(f"{phase}: {error}" for phase, error in errors))
def _deactivate_deployment_mutations(
@@ -285,30 +315,75 @@ def pending_tool_envs(manifest: DeploymentManifest) -> dict[str, dict[str, str]]
def _activate_deployment_mutations(manifest: DeploymentManifest) -> None:
for target, names in sorted(_reconcile_tool_envs(manifest).items()):
click.echo(f"Applying newer managed settings for {target}: {', '.join(names)}")
- manifest.mutations = apply_mutations(manifest)
- save_manifest(manifest)
+ try:
+ manifest.mutations = apply_mutations(manifest)
+ _save_apply_manifest(manifest)
+ except Exception as exc:
+ if manifest.mutations:
+ try:
+ revert_mutations(manifest)
+ except Exception as rollback_exc:
+ raise RuntimeError(
+ f"mutation activation failed: {exc}; rollback failed: {rollback_exc}"
+ ) from exc
+ else:
+ manifest.mutations = []
+ raise
+
+
+def _save_apply_manifest(manifest: DeploymentManifest) -> None:
+ """Use strict persistence for real manifests while keeping helper doubles light."""
+
+ if isinstance(manifest, DeploymentManifest):
+ save_manifest_strict(manifest)
+ else:
+ save_manifest(manifest)
+
+
+def _save_recovery_snapshot(manifest: DeploymentManifest, profile: str | None = None) -> None:
+ if isinstance(manifest, DeploymentManifest):
+ snapshot = deepcopy(manifest)
+ if profile is not None:
+ snapshot.profile = profile
+ save_recovery_manifest(snapshot)
+
+
+def _delete_recovery_snapshot(profile: str) -> None:
+ try:
+ delete_recovery_manifest(profile)
+ except Exception as exc:
+ raise click.ClickException(
+ f"Recovery snapshot {recovery_manifest_path(profile)} could not be deleted: {exc}. "
+ "It was retained; resolve the filesystem failure and retry."
+ ) from None
def _remove_deployment(manifest: DeploymentManifest) -> None:
+ errors: list[tuple[str, Exception]] = []
try:
_deactivate_deployment_mutations(manifest, persist_manifest=False)
- except Exception:
- pass
+ except Exception as exc:
+ errors.append(("mutation cleanup", exc))
try:
_stop_deployment(manifest)
- except Exception:
- pass
+ except Exception as exc:
+ errors.append(("owner stop", exc))
try:
remove_supervisor(manifest)
- except Exception:
- pass
- delete_manifest(manifest.profile)
+ except Exception as exc:
+ errors.append(("supervisor removal", exc))
+ if errors:
+ raise RuntimeError("; ".join(f"{phase}: {error}" for phase, error in errors))
+ try:
+ delete_manifest(manifest.profile)
+ except Exception as exc:
+ raise RuntimeError(f"manifest removal: {exc}") from exc
def _restore_deployment(manifest: DeploymentManifest) -> None:
restored = deepcopy(manifest)
- restored.artifacts = install_supervisor(restored)
- save_manifest(restored)
+ restored.artifacts = install_supervisor(restored, start=False)
+ _save_apply_manifest(restored)
_start_deployment(restored)
_activate_deployment_mutations(restored)
@@ -504,33 +579,99 @@ def _capture_passthrough_env(environ: Mapping[str, str]) -> dict[str, str]:
def _apply_manifest(manifest: DeploymentManifest) -> None:
+ profile = manifest.profile
+ recovery_saved = False
+ active_persistence_failed = False
+ existing = None
try:
- existing = load_manifest(manifest.profile)
- except ManifestError as e:
- # A corrupt existing manifest shouldn't block a fresh apply; overwrite it.
- click.echo(f"Warning: {e}; overwriting.")
- existing = None
- if existing is not None:
- click.echo(f"Updating existing deployment profile '{manifest.profile}'...")
- _remove_deployment(existing)
+ try:
+ existing = load_manifest(profile)
+ except ManifestError as e:
+ click.echo(f"Warning: {e}; overwriting.")
+ if existing is not None:
+ click.echo(f"Updating existing deployment profile '{profile}'...")
+ _save_recovery_snapshot(existing, profile)
+ recovery_saved = True
+ _remove_deployment(existing)
+ except Exception as exc:
+ recovery_detail = (
+ f" Recovery snapshot: {recovery_manifest_path(profile)} is retained; "
+ "no new owner was started."
+ if recovery_saved
+ else " No new owner was started."
+ )
+ raise click.ClickException(
+ f"Failed to prepare deployment '{profile}': {exc}.{recovery_detail}"
+ ) from exc
try:
- manifest.artifacts = install_supervisor(manifest)
- save_manifest(manifest)
+ try:
+ _save_apply_manifest(manifest)
+ except Exception:
+ active_persistence_failed = True
+ raise
+ manifest.artifacts = install_supervisor(manifest, start=False)
+ try:
+ _save_apply_manifest(manifest)
+ except Exception:
+ active_persistence_failed = True
+ raise
_start_deployment(manifest)
- _activate_deployment_mutations(manifest)
+ try:
+ _activate_deployment_mutations(manifest)
+ except Exception:
+ active_persistence_failed = True
+ raise
except Exception as exc:
- _remove_deployment(manifest)
- if existing is not None:
- click.echo(f"Restoring previous deployment '{manifest.profile}'...")
- _restore_deployment(existing)
- # Surface non-Click errors (OSError, CalledProcessError, ...) as a clean
- # message rather than a raw traceback; Click errors pass through as-is.
+ cleanup_errors: list[Exception] = []
+ try:
+ _remove_deployment(manifest)
+ except Exception as cleanup_exc:
+ cleanup_errors.append(cleanup_exc)
+ if not cleanup_errors and not active_persistence_failed and existing is not None:
+ click.echo(f"Restoring previous deployment '{profile}'...")
+ try:
+ _restore_deployment(existing)
+ except Exception as restore_error:
+ raise click.ClickException(
+ f"Failed to install deployment '{profile}': {exc}; "
+ f"previous deployment restoration also failed: {restore_error}; "
+ f"recovery snapshot: {recovery_manifest_path(profile)}; "
+ "restore it after resolving the failure"
+ ) from exc
+ _delete_recovery_snapshot(profile)
+
+ def _recovery_detail() -> str:
+ if recovery_saved:
+ return (
+ f"; recovery snapshot: {recovery_manifest_path(profile)}; "
+ "remove the new owner before restoring the snapshot"
+ )
+ return ""
+
+ cleanup_detail = ""
+ if cleanup_errors:
+ cleanup_detail = "; cleanup also failed: " + " | ".join(map(str, cleanup_errors))
+ persistence_detail = (
+ "; active manifest persistence failed; keep the recovery snapshot"
+ if active_persistence_failed and recovery_saved
+ else ""
+ )
if isinstance(exc, click.ClickException | click.Abort):
+ if cleanup_errors or persistence_detail:
+ raise click.ClickException(
+ f"Failed to install deployment '{profile}': {exc}"
+ f"{cleanup_detail}{persistence_detail}{_recovery_detail()}"
+ ) from exc
raise
- raise click.ClickException(
- f"Failed to install deployment '{manifest.profile}': {exc}"
- ) from exc
+ if cleanup_errors or persistence_detail:
+ raise click.ClickException(
+ f"Failed to install deployment '{profile}': {exc}"
+ f"{cleanup_detail}{persistence_detail}{_recovery_detail()}"
+ ) from exc
+ raise click.ClickException(f"Failed to install deployment '{profile}': {exc}") from exc
+ if recovery_saved:
+ _delete_recovery_snapshot(profile)
def _echo_installed(manifest: DeploymentManifest, *, prefix: str = "Installed persistent") -> None:
@@ -978,21 +1119,13 @@ def install_remove(profile: str) -> None:
"""Remove a persistent deployment and undo managed config."""
manifest = _require_manifest(profile)
- _deactivate_deployment_mutations(manifest, persist_manifest=False)
- try:
- if manifest.supervisor_kind == SupervisorKind.SERVICE.value:
- stop_supervisor(manifest)
- except Exception:
- pass
try:
- stop_runtime(manifest)
- except Exception:
- pass
- try:
- remove_supervisor(manifest)
- except Exception:
- pass
- delete_manifest(profile)
+ _remove_deployment(manifest)
+ except Exception as exc:
+ raise click.ClickException(
+ f"Failed to remove deployment '{profile}': cleanup failed: {exc}. "
+ "The deployment manifest was retained; resolve the cleanup failure and retry."
+ ) from exc
click.echo(f"Removed deployment '{profile}'.")
@@ -1019,7 +1152,7 @@ def install_agent_ensure(profile: str) -> None:
"""Ensure a persistent deployment is healthy, starting it when needed."""
manifest = _require_manifest(profile)
- if probe_ready(manifest.health_url):
+ if runtime_status(manifest) == "running" and probe_ready(manifest.health_url):
click.echo(f"Deployment '{profile}' is already healthy.")
return
with acquire_runtime_start_lock(manifest.profile) as acquired:
@@ -1028,13 +1161,15 @@ def install_agent_ensure(profile: str) -> None:
return
# Double-check after acquiring the lock — another ensure may have
# started the runtime while we waited for the lock.
- if probe_ready(manifest.health_url):
+ if runtime_status(manifest) == "running" and probe_ready(manifest.health_url):
click.echo(f"Deployment '{profile}' is already healthy.")
return
if runtime_status(manifest) == "running":
# Runtime exists but isn't ready yet — give it a grace period
# before deciding it's wedged and restarting.
- if wait_ready(manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS):
+ if wait_ready(
+ manifest, timeout_seconds=_STARTUP_READY_TIMEOUT_SECONDS, require_identity=True
+ ):
click.echo(f"Deployment '{profile}' is healthy.")
return
_deactivate_deployment_mutations(manifest)
diff --git a/headroom/cli/proxy.py b/headroom/cli/proxy.py
index e9ca230cb..ed416381e 100644
--- a/headroom/cli/proxy.py
+++ b/headroom/cli/proxy.py
@@ -221,6 +221,18 @@ def dashboard(port: int, no_open: bool) -> None:
@main.command()
+ "--headroom-deployment-profile",
+ hidden=True,
+ expose_value=False,
+ help="Internal persistent-deployment identity marker.",
+)
+ "--headroom-deployment-runtime",
+ hidden=True,
+ expose_value=False,
+ help="Internal persistent-deployment identity marker.",
+)
@click.option(
"--host",
default="127.0.0.1",
diff --git a/headroom/cli/wrap.py b/headroom/cli/wrap.py
index 27513b265..e7a017662 100644
--- a/headroom/cli/wrap.py
+++ b/headroom/cli/wrap.py
@@ -4393,22 +4393,40 @@ def _find_persistent_manifest(port: int) -> Any:
return manifests[0] if manifests else None
+def _wait_for_runtime_ready(manifest: Any, timeout_seconds: int) -> bool:
+ """Keep wrap recovery gated by both readiness and runtime identity."""
+ from headroom.install.runtime import wait_ready
+
+ try:
+ return wait_ready(manifest, timeout_seconds=timeout_seconds, require_identity=True)
+ except TypeError:
+ # Compatibility for test doubles that predate the keyword-only guard.
+ return wait_ready(manifest, timeout_seconds=timeout_seconds)
+
+
def _recover_persistent_proxy(port: int) -> bool:
"""Start or recover a matching persistent deployment for the requested port."""
- from headroom.install.health import probe_ready
- from headroom.install.models import InstallPreset, SupervisorKind
- from headroom.install.runtime import start_detached_agent, start_persistent_docker, wait_ready
+ from headroom.install.models import SupervisorKind
+ from headroom.install.runtime import (
+ runtime_ownership,
+ runtime_ready,
+ start_detached_agent,
+ start_persistent_docker,
+ )
from headroom.install.supervisors import start_supervisor
manifest = _find_persistent_manifest(port)
if manifest is None:
return False
- if probe_ready(manifest.health_url):
+ ownership = runtime_ownership(manifest)
+ if runtime_ready(manifest):
click.echo(f" Reusing persistent deployment '{manifest.profile}' on port {port}")
return True
- if manifest.supervisor_kind == SupervisorKind.TASK.value:
+ if ownership == "docker-supervisor":
+ click.echo(f" Recovering persistent deployment '{manifest.profile}' on port {port}...")
+ elif manifest.supervisor_kind == SupervisorKind.TASK.value:
click.echo(
f" Warning: task-based deployment '{manifest.profile}' cannot be auto-recovered via wrap"
)
@@ -4416,7 +4434,7 @@ def _recover_persistent_proxy(port: int) -> bool:
click.echo(f" Recovering persistent deployment '{manifest.profile}' on port {port}...")
try:
- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:
+ if ownership == "docker-supervisor":
start_persistent_docker(manifest)
elif manifest.supervisor_kind == SupervisorKind.SERVICE.value:
start_supervisor(manifest)
@@ -4428,7 +4446,7 @@ def _recover_persistent_proxy(port: int) -> bool:
)
return False
- if wait_ready(manifest, timeout_seconds=45):
+ if _wait_for_runtime_ready(manifest, 45):
click.echo(f" Recovered persistent deployment '{manifest.profile}' on port {port}")
return True
@@ -4438,12 +4456,12 @@ def _recover_persistent_proxy(port: int) -> bool:
def _restart_persistent_proxy(manifest: Any, port: int) -> bool:
"""Restart a persistent deployment after an idle stale-version detection."""
- from headroom.install.models import InstallPreset, SupervisorKind
+ from headroom.install.models import SupervisorKind
from headroom.install.runtime import (
+ runtime_ownership,
start_detached_agent,
start_persistent_docker,
stop_runtime,
- wait_ready,
)
from headroom.install.supervisors import start_supervisor
@@ -4452,7 +4470,7 @@ def _restart_persistent_proxy(manifest: Any, port: int) -> bool:
f"with Headroom {_HEADROOM_VERSION}..."
)
try:
- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:
+ if runtime_ownership(manifest) == "docker-supervisor":
stop_runtime(manifest)
start_persistent_docker(manifest)
elif manifest.supervisor_kind == SupervisorKind.SERVICE.value:
@@ -4468,7 +4486,7 @@ def _restart_persistent_proxy(manifest: Any, port: int) -> bool:
)
return False
- if wait_ready(manifest, timeout_seconds=45):
+ if _wait_for_runtime_ready(manifest, 45):
click.echo(f" Restarted persistent deployment '{manifest.profile}' on port {port}")
return True
@@ -4640,9 +4658,9 @@ def _ensure_proxy_unlocked(
)
manifest = None
if not isolated_copilot_subscription_proxy and manifest is not None:
- from headroom.install.health import probe_ready
+ from headroom.install.runtime import runtime_ready
- if probe_ready(manifest.health_url):
+ if runtime_ready(manifest):
health_payload = helpers._query_proxy_health(port)
running_config = helpers._proxy_health_config(health_payload)
if running_config is None:
diff --git a/headroom/install/paths.py b/headroom/install/paths.py
index ab536da97..8ef6992a0 100644
--- a/headroom/install/paths.py
+++ b/headroom/install/paths.py
@@ -40,6 +40,13 @@ def manifest_path(profile: str) -> Path:
return profile_root(profile) / "manifest.json"
+def recovery_manifest_path(profile: str) -> Path:
+ """Return the inactive recovery snapshot path for a named profile."""
+
+ validated = validate_profile_name(profile)
+ return deploy_root() / f"{validated}.recovery.json"
+
+
def log_path(profile: str) -> Path:
"""Return the log path used by persistent runner scripts."""
diff --git a/headroom/install/planner.py b/headroom/install/planner.py
index 6bdd3700b..27085b7bc 100644
--- a/headroom/install/planner.py
+++ b/headroom/install/planner.py
@@ -155,7 +155,11 @@ def build_manifest(
if sys.platform.startswith("win") and preset == InstallPreset.PERSISTENT_SERVICE.value:
effective_preset = InstallPreset.PERSISTENT_TASK.value
- if effective_preset == InstallPreset.PERSISTENT_SERVICE.value:
+ if runtime_kind == RuntimeKind.DOCKER.value:
+ # Docker owns its container lifecycle; never install a native service
+ # or task alongside it when callers combine runtime and preset flags.
+ supervisor_kind = SupervisorKind.NONE.value
+ elif effective_preset == InstallPreset.PERSISTENT_SERVICE.value:
supervisor_kind = SupervisorKind.SERVICE.value
elif effective_preset == InstallPreset.PERSISTENT_TASK.value:
supervisor_kind = SupervisorKind.TASK.value
@@ -226,6 +230,10 @@ def build_manifest(
"127.0.0.1",
"--port",
str(port),
+ "--headroom-deployment-profile",
+ normalized_profile,
+ "--headroom-deployment-runtime",
+ runtime_kind,
"--mode",
proxy_mode,
"--backend",
diff --git a/headroom/install/providers.py b/headroom/install/providers.py
index fd1fabfd8..9e25005f1 100644
--- a/headroom/install/providers.py
+++ b/headroom/install/providers.py
@@ -64,11 +64,33 @@ def _apply_unix_env_scope(manifest: DeploymentManifest) -> list[ManagedMutation]
else:
targets = unix_system_env_targets()
mutations: list[ManagedMutation] = []
- for path in targets:
- path.parent.mkdir(parents=True, exist_ok=True)
- merged = _merge_marker_block(path, block, _ENV_PATTERN, _ENV_MARKER_START)
- fsutil.write_text(path, merged)
- mutations.append(ManagedMutation(target="env", kind="shell-block", path=str(path)))
+ previous: list[tuple[Path, str | None]] = []
+ try:
+ for path in targets:
+ path.parent.mkdir(parents=True, exist_ok=True)
+ original = fsutil.read_text(path) if path.exists() else None
+ merged = _merge_marker_block(path, block, _ENV_PATTERN, _ENV_MARKER_START)
+ fsutil.write_text(path, merged)
+ # write_text is atomic, so only completed writes need restoring.
+ previous.append((path, original))
+ mutations.append(ManagedMutation(target="env", kind="shell-block", path=str(path)))
+ except Exception as exc:
+ # Restore prior contents so a reapply keeps the existing routing.
+ rollback_errors: list[str] = []
+ for path, content in reversed(previous):
+ try:
+ if content is None:
+ path.unlink(missing_ok=True)
+ else:
+ fsutil.write_text(path, content)
+ except Exception as rollback_exc:
+ rollback_errors.append(f"{path}: {rollback_exc}")
+ if rollback_errors:
+ raise RuntimeError(
+ f"shell environment update failed: {exc}; "
+ f"rollback failed: {'; '.join(rollback_errors)}"
+ ) from exc
+ raise
return mutations
@@ -89,37 +111,41 @@ def _apply_windows_env_scope(manifest: DeploymentManifest) -> list[ManagedMutati
scope_name = "Machine" if manifest.scope == ConfigScope.SYSTEM.value else "User"
merged = _unix_scope_values(manifest)
mutations: list[ManagedMutation] = []
- for name, value in merged.items():
- previous = run(
- [
+ try:
+ for name, value in merged.items():
+ previous = run(
+ [
+ "powershell",
+ "-NoProfile",
+ "-Command",
+ f"$value = [Environment]::GetEnvironmentVariable({_powershell_literal(name)},{_powershell_literal(scope_name)}); "
+ "if ($null -eq $value) { '__HEADROOM_UNSET__' } else { $value }",
+ ],
+ capture_output=True,
+ text=True,
+ check=True,
+ ).stdout.strip()
+ command = [
"powershell",
"-NoProfile",
"-Command",
- f"$value = [Environment]::GetEnvironmentVariable({_powershell_literal(name)},{_powershell_literal(scope_name)}); "
- "if ($null -eq $value) { '__HEADROOM_UNSET__' } else { $value }",
- ],
- capture_output=True,
- text=True,
- check=True,
- ).stdout.strip()
- command = [
- "powershell",
- "-NoProfile",
- "-Command",
- f"[Environment]::SetEnvironmentVariable({_powershell_literal(name)},{_powershell_literal(value)},{_powershell_literal(scope_name)})",
- ]
- subprocess.run(command, check=True)
- mutations.append(
- ManagedMutation(
- target="env",
- kind="windows-env",
- data={
- "name": name,
- "scope": scope_name,
- "previous": None if previous == "__HEADROOM_UNSET__" else previous,
- },
+ f"[Environment]::SetEnvironmentVariable({_powershell_literal(name)},{_powershell_literal(value)},{_powershell_literal(scope_name)})",
+ ]
+ subprocess.run(command, check=True)
+ mutations.append(
+ ManagedMutation(
+ target="env",
+ kind="windows-env",
+ data={
+ "name": name,
+ "scope": scope_name,
+ "previous": None if previous == "__HEADROOM_UNSET__" else previous,
+ },
+ )
)
- )
+ except Exception:
+ _remove_windows_env_scope(mutations)
+ raise
return mutations
@@ -151,15 +177,29 @@ def apply_mutations(manifest: DeploymentManifest) -> list[ManagedMutation]:
"""Apply provider/user/system configuration for a deployment."""
mutations: list[ManagedMutation] = []
- if manifest.scope in {ConfigScope.USER.value, ConfigScope.SYSTEM.value}:
- if os.name == "nt":
- mutations.extend(_apply_windows_env_scope(manifest))
- else:
- mutations.extend(_apply_unix_env_scope(manifest))
- mutations.extend(apply_provider_scope_mutations(manifest))
+ manifest.mutations = mutations
+ try:
+ if manifest.scope in {ConfigScope.USER.value, ConfigScope.SYSTEM.value}:
+ if os.name == "nt":
+ mutations.extend(_apply_windows_env_scope(manifest))
+ else:
+ mutations.extend(_apply_unix_env_scope(manifest))
+ # The registry already appends its records to manifest.mutations (this list).
+ for mutation in apply_provider_scope_mutations(manifest):
+ if not any(existing is mutation for existing in mutations):
+ mutations.append(mutation)
return mutations
-
- return [*mutations, *apply_provider_scope_mutations(manifest)]
+ except Exception as exc:
+ if mutations:
+ try:
+ revert_mutations(manifest)
+ except Exception as rollback_exc:
+ raise RuntimeError(
+ f"mutation application failed: {exc}; rollback failed: {rollback_exc}"
+ ) from exc
+ else:
+ manifest.mutations = []
+ raise
def revert_mutations(manifest: DeploymentManifest) -> None:
diff --git a/headroom/install/runtime.py b/headroom/install/runtime.py
index ce2bc5424..3be93ff76 100644
--- a/headroom/install/runtime.py
+++ b/headroom/install/runtime.py
@@ -3,6 +3,7 @@
from __future__ import annotations
import os
+import shlex
import shutil
import signal
import subprocess
@@ -21,6 +22,9 @@ from .models import DeploymentManifest, InstallPreset, RuntimeKind, SupervisorKi
from .paths import log_path, pid_path, profile_root
from .state import load_manifest
+_STOP_POLL_ATTEMPTS = 30
+_STOP_POLL_DELAY = 0.1
+
# Inside the container the proxy must listen on every interface so the
# host-side published port (127.0.0.1:<port>) can reach it. The proxy refuses
# a non-loopback bind with no HEADROOM_PROXY_TOKEN unless it is acknowledged;
@@ -213,6 +217,25 @@ def build_runtime_command(manifest: DeploymentManifest) -> list[str]:
return command
+def runtime_ownership(manifest: DeploymentManifest) -> str:
+ """Classify the owner that must launch and supervise this runtime."""
+
+ runtime_kind = getattr(manifest, "runtime_kind", None)
+ if (
+ runtime_kind == RuntimeKind.DOCKER.value
+ or manifest.preset == InstallPreset.PERSISTENT_DOCKER.value
+ ):
+ return "docker-supervisor"
+ if (
+ sys.platform == "darwin"
+ and manifest.preset == InstallPreset.PERSISTENT_SERVICE.value
+ and manifest.runtime_kind == RuntimeKind.PYTHON.value
+ and manifest.supervisor_kind == SupervisorKind.SERVICE.value
+ ):
+ return "launchd-exec"
+ return "popen"
+
+
def _write_pid(profile: str, pid: int) -> None:
path = pid_path(profile)
path.parent.mkdir(parents=True, exist_ok=True)
@@ -225,12 +248,154 @@ def _read_pid(profile: str) -> int | None:
return None
try:
return int(path.read_text().strip())
+ except (OSError, ValueError):
+ return None
+
+
+def _read_proc_metadata(pid: int) -> tuple[list[str], dict[str, str]] | None:
+ """Read process identity without requiring the optional psutil package."""
+
+ if Path(f"/proc/{pid}/cmdline").exists():
+ try:
+ cmdline = [
+ part.decode(errors="replace")
+ for part in Path(f"/proc/{pid}/cmdline").read_bytes().split(b"\0")
+ if part
+ ]
+ environ = {
+ part.split(b"=", 1)[0].decode(errors="replace"): part.split(b"=", 1)[1].decode(
+ errors="replace"
+ )
+ for part in Path(f"/proc/{pid}/environ").read_bytes().split(b"\0")
+ if b"=" in part
+ }
+ return cmdline, environ
+ except (OSError, ValueError):
+ return None
+
+ if _is_windows():
+ # WMI is part of Windows and exposes the full command line without
+ # requiring psutil. Environment blocks are not exposed reliably, so
+ # the command-line identity is checked together with the PID file.
+ try:
+ result = run(
+ [
+ "powershell.exe",
+ "-NoProfile",
+ "-NonInteractive",
+ "-Command",
+ f"(Get-CimInstance Win32_Process -Filter 'ProcessId = {pid}').CommandLine",
+ ],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ except (OSError, subprocess.SubprocessError):
+ return None
+ raw = result.stdout.strip()
+ if result.returncode != 0 or not raw:
+ return None
+ try:
+ return shlex.split(raw, posix=False), {}
+ except ValueError:
+ return None
+
+ # macOS does not expose /proc by default. `ps -wwE` is the supported
+ # system interface that returns the complete command and inherited env.
+ if sys.platform != "darwin":
+ return None
+ try:
+ result = run(
+ ["ps", "-wwE", "-p", str(pid), "-o", "command="],
+ capture_output=True,
+ text=True,
+ check=False,
+ )
+ except (OSError, subprocess.SubprocessError):
+ return None
+ raw = result.stdout.strip()
+ if result.returncode != 0 or not raw:
+ return None
+ try:
+ fields = shlex.split(raw)
except ValueError:
return None
+ environ = {}
+ for field in fields:
+ if field.startswith("HEADROOM_") and "=" in field:
+ name, value = field.split("=", 1)
+ environ[name] = value
+ return fields, environ
+
+
+def _proxy_command_matches(cmdline: list[str], manifest: DeploymentManifest) -> bool:
+ """Require the real Headroom proxy argv shape and configured port."""
+ shape = any(
+ cmdline[index : index + 3] == ["-m", "headroom.cli", "proxy"]
+ or cmdline[index : index + 2] == ["headroom", "proxy"]
+ for index in range(len(cmdline))
+ )
+ if not shape:
+ return False
+ for index, value in enumerate(cmdline):
+ if value == "--port" and index + 1 < len(cmdline):
+ return cmdline[index + 1] == str(manifest.port)
+ if value.startswith("--port="):
+ return value.split("=", 1)[1] == str(manifest.port)
+ return False
+
+
+def _proxy_deployment_markers_match(cmdline: list[str], manifest: DeploymentManifest) -> bool:
+ """Match the deployment identity embedded in supervisor-launched argv."""
+ markers = {
+ "--headroom-deployment-profile": manifest.profile,
+ "--headroom-deployment-runtime": manifest.runtime_kind,
+ }
+ for name, expected in markers.items():
+ if not any(
+ cmdline[index : index + 2] == [name, str(expected)] or value == f"{name}={expected}"
+ for index, value in enumerate(cmdline)
+ ):
+ return False
+ return True
-def _clear_pid(profile: str) -> None:
+def _process_matches_runtime(pid: int, manifest: DeploymentManifest) -> bool:
+ """Verify the PID's deployment identity before lifecycle operations."""
+ return _process_identity(pid, manifest) is True
+
+
+def _process_identity(pid: int, manifest: DeploymentManifest) -> bool | None:
+ """Return whether a PID matches, or ``None`` when identity is unavailable."""
+ if pid == os.getpid():
+ return False
+ try:
+ import psutil # type: ignore[import-untyped] # optional dependency
+
+ process = psutil.Process(pid)
+ cmdline = process.cmdline()
+ environ = process.environ()
+ except Exception:
+ metadata = _read_proc_metadata(pid)
+ if metadata is None:
+ return None
+ cmdline, environ = metadata
+ if not _proxy_command_matches(cmdline, manifest):
+ return False
+ if _is_windows() and not environ:
+ if not _proxy_deployment_markers_match(cmdline, manifest):
+ return None
+ return True
+ return bool(
+ environ.get("HEADROOM_DEPLOYMENT_PROFILE") == manifest.profile
+ and environ.get("HEADROOM_DEPLOYMENT_RUNTIME") == manifest.runtime_kind
+ )
+
+
+def _clear_pid(profile: str, *, expected_pid: int | None = None) -> None:
path = pid_path(profile)
+ if expected_pid is not None and _read_pid(profile) != expected_pid:
+ return
if path.exists():
path.unlink()
@@ -291,12 +456,26 @@ def acquire_runtime_start_lock(profile: str) -> Iterator[bool]:
def run_foreground(manifest: DeploymentManifest) -> int:
"""Run the raw runtime command in the foreground."""
+ if runtime_ownership(manifest) == "docker-supervisor":
+ raise RuntimeError(
+ "Docker deployments must be started by the Docker supervisor, not run_foreground"
+ )
command = build_runtime_command(manifest)
env = _runtime_env(manifest)
log_file_path = log_path(manifest.profile)
log_file_path.parent.mkdir(parents=True, exist_ok=True)
with open(log_file_path, "a", encoding="utf-8", errors="replace") as log_file:
+ if runtime_ownership(manifest) == "launchd-exec":
+ _write_pid(manifest.profile, os.getpid())
+ try:
+ os.dup2(log_file.fileno(), 1)
+ os.dup2(log_file.fileno(), 2)
+ os.execvpe(command[0], command, env)
+ except BaseException:
+ _clear_pid(manifest.profile, expected_pid=os.getpid())
+ raise
+
proc = subprocess.Popen(command, env=env, stdout=log_file, stderr=log_file)
_write_pid(manifest.profile, proc.pid)
@@ -313,7 +492,7 @@ def run_foreground(manifest: DeploymentManifest) -> int:
try:
return proc.wait()
finally:
- _clear_pid(manifest.profile)
+ _clear_pid(manifest.profile, expected_pid=proc.pid)
def start_detached_agent(profile: str) -> subprocess.Popen[str]:
@@ -359,51 +538,116 @@ def start_persistent_docker(manifest: DeploymentManifest) -> None:
manifest.container_name,
*command[5:], # drop initial `docker run --rm --name ...`
]
- run(
+ remove = run(
["docker", "rm", "-f", manifest.container_name],
capture_output=True,
text=True,
)
+ _require_docker_success(remove, "docker rm")
subprocess.run(docker_cmd, check=True)
def stop_runtime(manifest: DeploymentManifest) -> None:
"""Stop the raw runtime for the deployment."""
- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:
- run(
+ if runtime_ownership(manifest) == "docker-supervisor":
+ stop = run(
["docker", "stop", manifest.container_name],
capture_output=True,
text=True,
)
- run(
+ _require_docker_success(stop, "docker stop")
+ remove = run(
["docker", "rm", "-f", manifest.container_name],
capture_output=True,
text=True,
)
+ _require_docker_success(remove, "docker rm")
return
pid = _read_pid(manifest.profile)
if pid is None:
+ if pid_path(manifest.profile).exists():
+ raise RuntimeError(
+ f"Cannot stop deployment '{manifest.profile}': runner.pid is invalid"
+ )
+ return
+ if not pid_alive(pid):
+ _clear_pid(manifest.profile, expected_pid=pid)
+ return
+ identity = _process_identity(pid, manifest)
+ if identity is None:
+ raise RuntimeError(f"Cannot stop deployment '{manifest.profile}': runtime identity unknown")
+ if not identity:
+ _clear_pid(manifest.profile, expected_pid=pid)
return
try:
os.kill(pid, signal.SIGTERM)
except (OSError, SystemError):
# SystemError covers the Windows WinError 87 surfacing described in #1544.
pass
- _clear_pid(manifest.profile)
+ for _ in range(_STOP_POLL_ATTEMPTS):
+ if not pid_alive(pid):
+ if not pid_alive(pid):
+ _clear_pid(manifest.profile, expected_pid=pid)
+ return
+ raise RuntimeError(
+ f"Cannot stop deployment '{manifest.profile}': runtime identity changed"
+ )
+ if _process_identity(pid, manifest) is not True:
+ raise RuntimeError(
+ f"Cannot stop deployment '{manifest.profile}': runtime identity changed"
+ )
+ time.sleep(_STOP_POLL_DELAY)
+ raise RuntimeError(
+ f"Cannot stop deployment '{manifest.profile}': runtime remained alive after SIGTERM"
+ )
+
+
+def _command_output(result: Any) -> str:
+ return str(
+ getattr(result, "stderr", "") or getattr(result, "stdout", "") or "unknown error"
+ ).strip()
+
+def _require_docker_success(result: Any, operation: str) -> None:
+ if result.returncode == 0:
+ return
+ if result.returncode == 1 and "no such container" in _command_output(result).lower():
+ return
+ raise RuntimeError(f"{operation} failed: {_command_output(result)}")
-def wait_ready(manifest: DeploymentManifest, timeout_seconds: int = 30) -> bool:
+
+def wait_ready(
+ manifest: DeploymentManifest, timeout_seconds: int = 30, *, require_identity: bool = False
+) -> bool:
"""Wait for the deployment to report ready."""
for _ in range(timeout_seconds):
- if probe_ready(manifest.health_url):
+ if probe_ready(manifest.health_url) and (
+ not require_identity or runtime_status(manifest) == "running"
+ ):
return True
time.sleep(1)
return False
+def runtime_ready(manifest: DeploymentManifest) -> bool:
+ """Return ready only when health and deployment identity both match."""
+ from .health import probe_ready as current_probe_ready
+
+ if not hasattr(manifest, "runtime_kind"):
+ return bool(getattr(manifest, "health_url", None)) and current_probe_ready(
+ manifest.health_url
+ )
+ try:
+ return runtime_status(manifest) == "running" and current_probe_ready(manifest.health_url)
+ except AttributeError:
+ # Test doubles and old third-party callers may provide only the health
+ # contract; persisted DeploymentManifest instances always have identity.
+ return False
+
+
def wait_stopped(manifest: DeploymentManifest, timeout_seconds: int = 15) -> bool:
"""Wait for the deployment's health endpoint to stop answering."""
@@ -418,7 +662,7 @@ def wait_stopped(manifest: DeploymentManifest, timeout_seconds: int = 15) -> boo
def runtime_status(manifest: DeploymentManifest) -> str:
"""Return a short status string for the deployment runtime."""
- if manifest.preset == InstallPreset.PERSISTENT_DOCKER.value:
+ if runtime_ownership(manifest) == "docker-supervisor":
result = run(
["docker", "ps", "--format", "{{.Names}}"],
capture_output=True,
@@ -429,11 +673,22 @@ def runtime_status(manifest: DeploymentManifest) -> str:
return "stopped"
pid = _read_pid(manifest.profile)
if pid is None:
+ if pid_path(manifest.profile).exists():
+ return "unknown"
return "stopped"
# Windows-safe liveness probe: a bare os.kill(pid, 0) here raised WinError 87
# as a SystemError against the detached agent, crashing status and taking the
# live proxy down with it (#1544).
- return "running" if pid_alive(pid) else "stopped"
+ if not pid_alive(pid):
+ _clear_pid(manifest.profile, expected_pid=pid)
+ return "stopped"
+ identity = _process_identity(pid, manifest)
+ if identity is None:
+ return "unknown"
+ if not identity:
+ _clear_pid(manifest.profile, expected_pid=pid)
+ return "stopped"
+ return "running"
def detect_current_deployment() -> tuple[DeploymentManifest | None, str]:
@@ -456,7 +711,8 @@ def detect_current_deployment() -> tuple[DeploymentManifest | None, str]:
if not profile:
return None, "foreground"
manifest = load_manifest(profile)
- if preset == InstallPreset.PERSISTENT_DOCKER.value:
+ runtime = os.environ.get("HEADROOM_DEPLOYMENT_RUNTIME")
+ if preset == InstallPreset.PERSISTENT_DOCKER.value or runtime == RuntimeKind.DOCKER.value:
return manifest, "docker"
if manifest is None:
return None, "foreground"
diff --git a/headroom/install/state.py b/headroom/install/state.py
index 16f66a897..5a091a87a 100644
--- a/headroom/install/state.py
+++ b/headroom/install/state.py
@@ -12,7 +12,7 @@ from pathlib import Path
from typing import Any
from .models import ArtifactRecord, DeploymentManifest, ManagedMutation, iso_utc_now
-from .paths import deploy_root, manifest_path, profile_root
+from .paths import deploy_root, manifest_path, profile_root, recovery_manifest_path
logger = logging.getLogger(__name__)
@@ -34,7 +34,7 @@ def _atomic_write_text(path: Path, data: str) -> None:
fd, tmp_name = tempfile.mkstemp(dir=directory, prefix=f".{path.name}.", suffix=".tmp")
tmp_path = Path(tmp_name)
try:
- with os.fdopen(fd, "w", encoding="utf-8") as handle:
+ with os.fdopen(fd, "w", encoding="utf-8", newline="\n") as handle:
handle.write(data)
handle.flush()
os.fsync(handle.fileno())
@@ -44,6 +44,22 @@ def _atomic_write_text(path: Path, data: str) -> None:
raise
+def _manifest_json(manifest: DeploymentManifest) -> str:
+ return json.dumps(asdict(manifest), indent=2) + "\n"
+
+
+def _save_manifest_at(path: Path, manifest: DeploymentManifest, *, strict: bool) -> None:
+ root = path.parent
+ root.mkdir(parents=True, exist_ok=True)
+ manifest.updated_at = iso_utc_now()
+ try:
+ _atomic_write_text(path, _manifest_json(manifest))
+ except OSError:
+ if strict:
+ raise
+ logger.warning("Cannot save deployment manifest: %s — continuing without persistence", path)
+
+
def save_manifest(manifest: DeploymentManifest) -> None:
"""Persist a deployment manifest to disk.
@@ -52,15 +68,29 @@ def save_manifest(manifest: DeploymentManifest) -> None:
read-only filesystems by logging a warning instead of crashing.
"""
try:
- root = profile_root(manifest.profile)
- root.mkdir(parents=True, exist_ok=True)
- manifest.updated_at = iso_utc_now()
- path = manifest_path(manifest.profile)
- _atomic_write_text(path, json.dumps(asdict(manifest), indent=2) + "\n")
+ _save_manifest_at(manifest_path(manifest.profile), manifest, strict=False)
except OSError as e:
logger.warning("Cannot save deployment manifest: %s — continuing without persistence", e)
+def save_manifest_strict(manifest: DeploymentManifest) -> None:
+ """Persist a manifest and propagate errors when recovery depends on it."""
+
+ _save_manifest_at(manifest_path(manifest.profile), manifest, strict=True)
+
+
+def save_recovery_manifest(manifest: DeploymentManifest) -> None:
+ """Persist an inactive, complete snapshot for failed deployment recovery."""
+
+ _save_manifest_at(recovery_manifest_path(manifest.profile), manifest, strict=True)
+
+
+def delete_recovery_manifest(profile: str = "default") -> None:
+ """Delete an inactive recovery snapshot when its transaction is complete."""
+
+ recovery_manifest_path(profile).unlink(missing_ok=True)
+
+
# The Docker image org moved from a personal repo to the project org. The old
# ``ghcr.io/chopratejas/headroom`` repo is frozen at 0.27.0, so a manifest that
# still pins it silently runs ~5 minor versions behind the CLI with no drift
@@ -127,4 +157,4 @@ def delete_manifest(profile: str = "default") -> None:
root = profile_root(profile)
if root.exists():
- shutil.rmtree(root, ignore_errors=True)
+ shutil.rmtree(root)
diff --git a/headroom/install/supervisors.py b/headroom/install/supervisors.py
index 7591c1b18..b0db7fe57 100644
--- a/headroom/install/supervisors.py
+++ b/headroom/install/supervisors.py
@@ -445,7 +445,7 @@ def _cleanup_windows_tasks_for_fallback(names: list[str], *, must_delete: set[st
)
-def install_supervisor(manifest: DeploymentManifest) -> list[ArtifactRecord]:
+def install_supervisor(manifest: DeploymentManifest, *, start: bool = True) -> list[ArtifactRecord]:
"""Install service/task artifacts for the deployment."""
records = render_runner_scripts(manifest)
@@ -522,7 +522,8 @@ def install_supervisor(manifest: DeploymentManifest) -> list[ArtifactRecord]:
and manifest.supervisor_kind == SupervisorKind.SERVICE.value
else f"gui/{os.getuid()}"
)
- _bootstrap_with_retry(bootstrap_domain, plist_path)
+ if start:
+ _bootstrap_with_retry(bootstrap_domain, plist_path)
records.append(ArtifactRecord(kind="plist", path=str(plist_path)))
return records
@@ -661,6 +662,8 @@ def stop_supervisor(manifest: DeploymentManifest) -> None:
if manifest.supervisor_kind == SupervisorKind.NONE.value:
return
if sys.platform.startswith("linux"):
+ if manifest.supervisor_kind == SupervisorKind.TASK.value:
+ return
flags = [] if manifest.scope == "system" else ["--user"]
subprocess.run(["systemctl", *flags, "stop", manifest.service_name], check=True)
return
@@ -701,19 +704,21 @@ def remove_supervisor(manifest: DeploymentManifest) -> None:
if sys.platform.startswith("linux"):
if manifest.supervisor_kind == SupervisorKind.SERVICE.value:
flags = [] if manifest.scope == "system" else ["--user"]
- run(
+ result = run(
["systemctl", *flags, "disable", "--now", manifest.service_name],
capture_output=True,
text=True,
)
+ _require_removal_success(result, "systemctl disable --now", absent_text="not loaded")
unit_path, _ = _linux_service_unit(manifest, unix_run_script_path(manifest.profile))
if unit_path.exists():
unit_path.unlink()
- run(
+ result = run(
["systemctl", *flags, "daemon-reload"],
capture_output=True,
text=True,
)
+ _require_removal_success(result, "systemctl daemon-reload")
return
cron_path, _ = _linux_task_spec(manifest, unix_ensure_script_path(manifest.profile))
if cron_path and cron_path.exists():
@@ -725,17 +730,20 @@ def remove_supervisor(manifest: DeploymentManifest) -> None:
text=True,
)
if current.returncode != 0:
+ if "no crontab for" not in _result_text(current).lower():
+ raise click.ClickException(f"crontab -l failed: {_result_text(current)}")
return
marker_start = f"# >>> headroom {manifest.profile} >>>"
marker_end = f"# <<< headroom {manifest.profile} <<<"
pattern = re.compile(re.escape(marker_start) + r".*?" + re.escape(marker_end), re.DOTALL)
content = pattern.sub("", current.stdout).strip()
- run(
+ result = run(
["crontab", "-"],
input=(content + "\n") if content else "",
text=True,
check=True,
)
+ _require_removal_success(result, "crontab update")
return
if sys.platform == "darwin":
@@ -753,27 +761,64 @@ def remove_supervisor(manifest: DeploymentManifest) -> None:
and manifest.supervisor_kind == SupervisorKind.SERVICE.value
else f"gui/{os.getuid()}"
)
- run(
+ result = run(
["launchctl", "bootout", f"{domain}/{label}"],
capture_output=True,
text=True,
)
+ _require_removal_success(result, "launchctl bootout", absent_codes={_LAUNCHCTL_ESRCH})
if plist_path.exists():
plist_path.unlink()
return
if _is_windows():
if manifest.supervisor_kind == SupervisorKind.SERVICE.value:
- run(
+ result = run(
["sc.exe", "stop", manifest.service_name],
capture_output=True,
text=True,
)
- run(
+ _require_removal_success(result, "sc.exe stop", absent_text="does not exist")
+ result = run(
["sc.exe", "delete", manifest.service_name],
capture_output=True,
text=True,
)
+ _require_removal_success(result, "sc.exe delete", absent_text="does not exist")
return
- _delete_windows_task(f"{manifest.service_name}-startup")
- _delete_windows_task(f"{manifest.service_name}-health")
+ result = run(
+ ["schtasks", "/Delete", "/TN", f"{manifest.service_name}-startup", "/F"],
+ capture_output=True,
+ text=True,
+ )
+ _require_removal_success(result, "schtasks startup delete", absent_text="does not exist")
+ result = run(
+ ["schtasks", "/Delete", "/TN", f"{manifest.service_name}-health", "/F"],
+ capture_output=True,
+ text=True,
+ )
+ _require_removal_success(result, "schtasks health delete", absent_text="does not exist")
+
+
+def _result_text(result: object) -> str:
+ return str(
+ getattr(result, "stderr", "") or getattr(result, "stdout", "") or "unknown error"
+ ).strip()
+
+
+def _require_removal_success(
+ result: object,
+ operation: str,
+ *,
+ absent_codes: set[int] | None = None,
+ absent_text: str | None = None,
+) -> None:
+ returncode = int(getattr(result, "returncode", 0))
+ detail = _result_text(result)
+ if returncode == 0:
+ return
+ if absent_codes and returncode in absent_codes:
+ return
+ if absent_text and absent_text in detail.lower():
+ return
+ raise click.ClickException(f"{operation} failed: {detail}")
diff --git a/headroom/providers/install_registry.py b/headroom/providers/install_registry.py
index a3e053aa1..a998bed90 100644
--- a/headroom/providers/install_registry.py
+++ b/headroom/providers/install_registry.py
@@ -85,13 +85,33 @@ def build_install_target_envs(
def apply_provider_scope_mutations(manifest: DeploymentManifest) -> list[ManagedMutation]:
"""Apply provider-scope mutations owned by provider slices."""
mutations: list[ManagedMutation] = []
- for target in manifest.targets:
- handlers = _PROVIDER_SCOPE_HANDLERS.get(target)
- if handlers is None:
- continue
- mutation = handlers[0](manifest)
- if mutation is not None:
- mutations.append(mutation)
+ tracked = getattr(manifest, "mutations", None)
+ try:
+ for target in manifest.targets:
+ handlers = _PROVIDER_SCOPE_HANDLERS.get(target)
+ if handlers is None:
+ continue
+ mutation = handlers[0](manifest)
+ if mutation is not None:
+ mutations.append(mutation)
+ if tracked is not None and tracked is not mutations:
+ tracked.append(mutation)
+ except Exception as exc:
+ rollback_errors: list[Exception] = []
+ for mutation in reversed(mutations):
+ try:
+ revert_provider_scope_mutation(manifest, mutation)
+ except Exception as rollback_exc:
+ rollback_errors.append(rollback_exc)
+ else:
+ if tracked is not None and mutation in tracked:
+ tracked.remove(mutation)
+ if rollback_errors:
+ details = "; ".join(str(error) for error in rollback_errors)
+ raise RuntimeError(
+ f"provider mutation failed: {exc}; rollback failed: {details}"
+ ) from exc
+ raise
return mutations
diff --git a/tests/test_cli/test_install_cli.py b/tests/test_cli/test_install_cli.py
index 1fd12c251..c392d6b82 100644
--- a/tests/test_cli/test_install_cli.py
+++ b/tests/test_cli/test_install_cli.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+import json
+from pathlib import Path
from types import SimpleNamespace
import click
@@ -8,7 +10,9 @@ from click.testing import CliRunner
from headroom.cli import install as inst
from headroom.cli.main import main
+from headroom.install.models import DeploymentManifest, ManagedMutation
from headroom.install.planner import build_tool_envs
+from headroom.install.state import load_manifest as load_state_manifest
def test_require_manifest_resolves_single_profile_when_default_missing(monkeypatch):
@@ -148,7 +152,7 @@ def test_install_apply_starts_service_supervisor(monkeypatch) -> None:
"headroom.cli.install.apply_mutations",
lambda deployment: calls.append("apply") or [],
)
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr(
"headroom.cli.install.save_manifest", lambda deployment: calls.append("save")
)
@@ -163,7 +167,7 @@ def test_install_apply_starts_service_supervisor(monkeypatch) -> None:
lambda deployment: calls.append("start_docker"),
)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")
@@ -173,7 +177,7 @@ def test_install_apply_starts_service_supervisor(monkeypatch) -> None:
assert result.exit_code == 0, result.output
assert "Installed persistent deployment 'default'" in result.output
assert "Targets: claude, codex" in result.output
- assert calls == ["save", "start_service", "apply", "save"]
+ assert calls == ["save", "save", "start_service", "apply", "save"]
def test_install_apply_announces_windows_service_fallback(monkeypatch) -> None:
@@ -197,7 +201,7 @@ def test_install_apply_announces_windows_service_fallback(monkeypatch) -> None:
monkeypatch.setattr("headroom.cli.install._is_windows", lambda: True)
monkeypatch.setattr("headroom.cli.install.build_manifest", lambda **_: Manifest())
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
@@ -206,7 +210,7 @@ def test_install_apply_announces_windows_service_fallback(monkeypatch) -> None:
"headroom.cli.install.start_detached_agent", lambda profile: calls.append("start_agent")
)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
result = runner.invoke(main, ["install", "apply", "--preset", "persistent-service"])
@@ -246,12 +250,12 @@ def test_install_apply_forwards_no_http2_to_build_manifest(monkeypatch) -> None:
monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build_manifest)
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.start_detached_agent", lambda profile: None)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
result = runner.invoke(main, ["install", "apply", "--no-http2"])
@@ -284,12 +288,12 @@ def _patch_apply_pipeline(monkeypatch, captured: dict[str, object]):
monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build_manifest)
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.start_detached_agent", lambda profile: None)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
@@ -409,12 +413,12 @@ def _apply_capturing_build_manifest(monkeypatch) -> dict[str, object]:
monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build_manifest)
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.start_detached_agent", lambda profile: None)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
return captured
@@ -461,7 +465,10 @@ def test_install_status_includes_backend_from_health_probe(monkeypatch) -> None:
health_url = "http://127.0.0.1:8787/readyz"
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())
- monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")
+ monkeypatch.setattr(
+ "headroom.cli.install.runtime_status",
+ lambda manifest: "running",
+ )
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: True)
monkeypatch.setattr(
"headroom.cli.install.probe_json",
@@ -536,7 +543,7 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:
"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45, **kwargs: True
)
monkeypatch.setattr(
"headroom.cli.install.apply_mutations", lambda manifest: calls.append("apply") or []
@@ -561,6 +568,100 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:
]
[email protected]("supervisor_kind", ["service", "task"])
+def test_stop_deployment_stops_external_supervisor_before_docker(
+ monkeypatch, supervisor_kind: str
+) -> None:
+ calls: list[str] = []
+ manifest = SimpleNamespace(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="docker",
+ supervisor_kind=supervisor_kind,
+ health_url="http://127.0.0.1:8787/readyz",
+ )
+ monkeypatch.setattr(inst, "stop_supervisor", lambda current: calls.append("supervisor"))
+ monkeypatch.setattr(inst, "stop_runtime", lambda current: calls.append("runtime"))
+ monkeypatch.setattr(inst, "wait_stopped", lambda current: calls.append("wait_stopped") or True)
+
+ inst._stop_deployment(manifest)
+
+ assert calls == ["supervisor", "runtime", "wait_stopped"]
+
+
+def test_stop_deployment_stops_docker_even_when_supervisor_stop_fails(monkeypatch) -> None:
+ calls: list[str] = []
+ manifest = SimpleNamespace(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="docker",
+ supervisor_kind="service",
+ )
+
+ def fail_supervisor(current):
+ calls.append("supervisor")
+ raise RuntimeError("supervisor unavailable")
+
+ monkeypatch.setattr(inst, "stop_supervisor", fail_supervisor)
+ monkeypatch.setattr(inst, "stop_runtime", lambda current: calls.append("runtime"))
+
+ with pytest.raises(RuntimeError, match="supervisor unavailable"):
+ inst._stop_deployment(manifest)
+ assert calls == ["supervisor", "runtime"]
+
+
+def test_remove_deployment_retains_manifest_when_supervisor_stop_fails(monkeypatch) -> None:
+ calls: list[str] = []
+ manifest = SimpleNamespace(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="docker",
+ supervisor_kind="service",
+ mutations=[],
+ )
+ monkeypatch.setattr(
+ inst,
+ "stop_supervisor",
+ lambda current: (_ for _ in ()).throw(RuntimeError("supervisor unavailable")),
+ )
+ monkeypatch.setattr(inst, "stop_runtime", lambda current: calls.append("runtime"))
+ monkeypatch.setattr(
+ inst, "remove_supervisor", lambda current: calls.append("remove-supervisor")
+ )
+ monkeypatch.setattr(inst, "delete_manifest", lambda profile: calls.append("delete"))
+
+ with pytest.raises(RuntimeError, match="supervisor unavailable"):
+ inst._remove_deployment(manifest)
+ assert calls == ["runtime", "remove-supervisor"]
+
+
+def test_remove_deployment_reports_all_cleanup_failures(monkeypatch) -> None:
+ manifest = SimpleNamespace(profile="default", mutations=[object()])
+ monkeypatch.setattr(
+ inst,
+ "_deactivate_deployment_mutations",
+ lambda *args, **kwargs: (_ for _ in ()).throw(RuntimeError("mutation failed")),
+ )
+ monkeypatch.setattr(
+ inst,
+ "_stop_deployment",
+ lambda current: (_ for _ in ()).throw(RuntimeError("runtime failed")),
+ )
+ monkeypatch.setattr(
+ inst,
+ "remove_supervisor",
+ lambda current: (_ for _ in ()).throw(RuntimeError("supervisor failed")),
+ )
+
+ with pytest.raises(RuntimeError) as exc:
+ inst._remove_deployment(manifest)
+
+ message = str(exc.value)
+ assert all(
+ item in message for item in ("mutation failed", "runtime failed", "supervisor failed")
+ )
+
+
def _restart_race_manifest():
class Manifest:
profile = "default"
@@ -602,7 +703,8 @@ def test_install_restart_waits_for_old_process_before_starting(monkeypatch) -> N
"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready",
+ lambda manifest, timeout_seconds=45, **kwargs: True,
)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda manifest: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: None)
@@ -659,6 +761,7 @@ def test_install_start_noops_when_already_healthy(monkeypatch) -> None:
mutations = [object()]
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())
+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: True)
monkeypatch.setattr(
"headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
@@ -671,6 +774,28 @@ def test_install_start_noops_when_already_healthy(monkeypatch) -> None:
assert calls == []
+def test_start_deployment_requires_identity_for_post_start_readiness(monkeypatch) -> None:
+ manifest = SimpleNamespace(
+ profile="default",
+ preset="persistent-task",
+ runtime_kind="python",
+ supervisor_kind="none",
+ health_url="http://127.0.0.1:8787/readyz",
+ )
+ waits: list[dict[str, object]] = []
+ monkeypatch.setattr(inst, "runtime_status", lambda current: "stopped")
+ monkeypatch.setattr(inst, "start_detached_agent", lambda profile: None)
+ monkeypatch.setattr(
+ inst,
+ "wait_ready",
+ lambda current, timeout_seconds, **kwargs: waits.append(kwargs) or True,
+ )
+
+ inst._start_deployment(manifest, assume_start_lock=True)
+
+ assert waits == [{"require_identity": True}]
+
+
def test_install_start_noops_for_healthy_docker_without_docker_on_path(monkeypatch) -> None:
runner = CliRunner()
@@ -688,13 +813,14 @@ def test_install_start_noops_for_healthy_docker_without_docker_on_path(monkeypat
mutations = [object()]
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())
+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: True)
monkeypatch.setattr("headroom.cli.install.shutil.which", lambda name, *args, **kwargs: None)
result = runner.invoke(main, ["install", "start"])
- assert result.exit_code == 0, result.output
- assert "Started deployment 'default'." in result.output
+ assert result.exit_code != 0
+ assert "docker' was not found on PATH" in result.output
def test_install_start_does_not_spawn_when_start_lock_is_contended(monkeypatch) -> None:
@@ -763,7 +889,8 @@ def test_install_start_restarts_wedged_runtime_under_single_lock(monkeypatch) ->
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")
wait_results = iter([False, True])
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds: next(wait_results)
+ "headroom.cli.install.wait_ready",
+ lambda manifest, timeout_seconds, **kwargs: next(wait_results),
)
monkeypatch.setattr(
"headroom.cli.install.revert_mutations", lambda manifest: calls.append("revert")
@@ -831,12 +958,12 @@ def test_install_apply_accepts_opencode_target(monkeypatch) -> None:
monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build_manifest)
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.start_detached_agent", lambda profile: None)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
result = runner.invoke(
@@ -891,7 +1018,9 @@ def test_install_apply_restores_previous_deployment_after_failed_update(monkeypa
)
monkeypatch.setattr(
"headroom.cli.install.install_supervisor",
- lambda deployment: calls.append(f"supervisor:{','.join(deployment.targets)}") or [],
+ lambda deployment, **kwargs: (
+ calls.append(f"supervisor:{','.join(deployment.targets)}") or []
+ ),
)
monkeypatch.setattr(
"headroom.cli.install.save_manifest",
@@ -917,6 +1046,10 @@ def test_install_apply_restores_previous_deployment_after_failed_update(monkeypa
"headroom.cli.install.delete_manifest",
lambda profile: calls.append(f"delete:{profile}"),
)
+ monkeypatch.setattr(
+ "headroom.cli.install.delete_recovery_manifest",
+ lambda profile: calls.append(f"delete-recovery:{profile}"),
+ )
def _start(deployment) -> None:
calls.append(f"start:{','.join(deployment.targets)}")
@@ -935,6 +1068,7 @@ def test_install_apply_restores_previous_deployment_after_failed_update(monkeypa
"stop-runtime:codex",
"remove-supervisor:codex",
"delete:default",
+ "save:claude",
"supervisor:claude",
"save:claude",
"start:claude",
@@ -947,9 +1081,399 @@ def test_install_apply_restores_previous_deployment_after_failed_update(monkeypa
"start:codex",
"apply:codex",
"save:codex",
+ "delete-recovery:default",
]
+def test_install_apply_reports_restore_failure_with_recovery_path(monkeypatch) -> None:
+ previous = SimpleNamespace(profile="default")
+ new = SimpleNamespace(profile="default", artifacts=[], mutations=[])
+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)
+ monkeypatch.setattr(inst, "_save_recovery_snapshot", lambda *args: None)
+ monkeypatch.setattr(inst, "_remove_deployment", lambda manifest: None)
+ monkeypatch.setattr(inst, "_save_apply_manifest", lambda manifest: None)
+ monkeypatch.setattr(inst, "install_supervisor", lambda manifest, **kwargs: [])
+ monkeypatch.setattr(
+ inst,
+ "_start_deployment",
+ lambda manifest: (_ for _ in ()).throw(click.ClickException("startup failed")),
+ )
+ monkeypatch.setattr(
+ inst,
+ "_restore_deployment",
+ lambda manifest: (_ for _ in ()).throw(RuntimeError("restore failed")),
+ )
+
+ with pytest.raises(click.ClickException) as exc:
+ inst._apply_manifest(new)
+
+ message = str(exc.value)
+ assert "restore failed" in message
+ assert "recovery snapshot:" in message
+ assert "restore it after resolving the failure" in message
+
+
+def test_install_apply_old_removal_failure_is_actionable_and_does_not_start_new(
+ monkeypatch,
+) -> None:
+ previous = SimpleNamespace(profile="default")
+ new = SimpleNamespace(profile="default", artifacts=[], mutations=[])
+ calls: list[str] = []
+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)
+ monkeypatch.setattr(inst, "_save_recovery_snapshot", lambda *args: calls.append("snapshot"))
+ monkeypatch.setattr(
+ inst,
+ "_remove_deployment",
+ lambda manifest: (_ for _ in ()).throw(RuntimeError("old removal failed")),
+ )
+ monkeypatch.setattr(inst, "_start_deployment", lambda manifest: calls.append("start"))
+
+ with pytest.raises(click.ClickException) as exc:
+ inst._apply_manifest(new)
+
+ assert "old removal failed" in str(exc.value)
+ assert "no new owner was started" in str(exc.value)
+ assert calls == ["snapshot"]
+
+
+def test_install_apply_uses_requested_profile_for_recovery_snapshot(
+ monkeypatch, tmp_path: Path
+) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ previous = DeploymentManifest(
+ profile="loaded-profile",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=["old"],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ new = DeploymentManifest(
+ profile="requested-profile",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=["new"],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)
+ monkeypatch.setattr(inst, "_remove_deployment", lambda manifest: None)
+ monkeypatch.setattr(inst, "_save_apply_manifest", lambda manifest: None)
+ monkeypatch.setattr(inst, "install_supervisor", lambda manifest, **kwargs: [])
+ monkeypatch.setattr(
+ inst,
+ "_start_deployment",
+ lambda manifest: (_ for _ in ()).throw(click.ClickException("startup failed")),
+ )
+
+ with pytest.raises(click.ClickException):
+ inst._apply_manifest(new)
+
+ assert (tmp_path / ".headroom" / "deploy" / "requested-profile.recovery.json").exists()
+ assert not (tmp_path / ".headroom" / "deploy" / "loaded-profile.recovery.json").exists()
+
+
+def test_restore_deployment_requires_durable_manifest_before_start(monkeypatch) -> None:
+ manifest = DeploymentManifest(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=[],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ calls: list[str] = []
+ monkeypatch.setattr(inst, "install_supervisor", lambda current, **kwargs: [])
+ monkeypatch.setattr(
+ inst,
+ "_save_apply_manifest",
+ lambda current: (_ for _ in ()).throw(OSError("restore manifest busy")),
+ )
+ monkeypatch.setattr(inst, "_start_deployment", lambda current: calls.append("start"))
+
+ with pytest.raises(OSError, match="restore manifest busy"):
+ inst._restore_deployment(manifest)
+ assert calls == []
+
+
+def test_apply_persists_before_darwin_bootstrap_or_start_on_save_failure(monkeypatch) -> None:
+ manifest = DeploymentManifest(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=[],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ install_calls: list[bool] = []
+ bootstrap_calls: list[str] = []
+ start_calls: list[str] = []
+ save_calls = 0
+
+ def save(current) -> None:
+ nonlocal save_calls
+ save_calls += 1
+ if save_calls == 2:
+ raise OSError("active manifest busy")
+
+ def install(current, *, start=True):
+ install_calls.append(start)
+ if start:
+ bootstrap_calls.append(current.profile)
+ return []
+
+ monkeypatch.setattr(inst, "load_manifest", lambda profile: None)
+ monkeypatch.setattr(inst, "_save_apply_manifest", save)
+ monkeypatch.setattr(inst, "install_supervisor", install)
+ monkeypatch.setattr(
+ inst, "_start_deployment", lambda current: start_calls.append(current.profile)
+ )
+ monkeypatch.setattr(inst, "_remove_deployment", lambda current: None)
+
+ with pytest.raises(click.ClickException, match="active manifest busy"):
+ inst._apply_manifest(manifest)
+
+ assert install_calls == [False]
+ assert bootstrap_calls == []
+ assert start_calls == []
+
+
+def test_activate_mutations_reverts_side_effects_when_strict_save_fails(monkeypatch) -> None:
+ manifest = DeploymentManifest(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=[],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ mutation = ManagedMutation(target="env", kind="shell-block", path="settings")
+ reverted: list[ManagedMutation] = []
+ monkeypatch.setattr(inst, "apply_mutations", lambda current: [mutation])
+ monkeypatch.setattr(
+ inst,
+ "_save_apply_manifest",
+ lambda current: (_ for _ in ()).throw(OSError("manifest busy")),
+ )
+ monkeypatch.setattr(
+ inst, "revert_mutations", lambda current: reverted.extend(current.mutations)
+ )
+
+ with pytest.raises(OSError, match="manifest busy"):
+ inst._activate_deployment_mutations(manifest)
+ assert reverted == [mutation]
+ assert manifest.mutations == []
+
+
+def test_recovery_snapshot_delete_failure_is_actionable(monkeypatch) -> None:
+ monkeypatch.setattr(
+ inst,
+ "delete_recovery_manifest",
+ lambda profile: (_ for _ in ()).throw(OSError("snapshot busy")),
+ )
+
+ with pytest.raises(click.ClickException, match="could not be deleted") as exc:
+ inst._delete_recovery_snapshot("default")
+ assert "snapshot busy" in str(exc.value)
+ assert "retained" in str(exc.value)
+
+
+def test_install_apply_keeps_new_owner_recovery_fail_closed(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ previous = DeploymentManifest(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=["old"],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ new = DeploymentManifest(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=["new"],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ calls: list[str] = []
+
+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)
+ real_save_recovery = inst._save_recovery_snapshot
+
+ def save_recovery(manifest, profile):
+ real_save_recovery(manifest, profile)
+ calls.append("snapshot")
+
+ monkeypatch.setattr(inst, "_save_recovery_snapshot", save_recovery)
+
+ def remove(manifest):
+ calls.append("remove-new" if manifest is new else "remove-old")
+ if manifest is new:
+ raise RuntimeError("new deployment cleanup failed")
+
+ monkeypatch.setattr(inst, "_remove_deployment", remove)
+ real_save_apply = inst._save_apply_manifest
+
+ def save_apply(manifest):
+ real_save_apply(manifest)
+ calls.append("save-new")
+
+ monkeypatch.setattr(inst, "_save_apply_manifest", save_apply)
+ monkeypatch.setattr(
+ inst, "install_supervisor", lambda manifest, **kwargs: calls.append("install-new") or []
+ )
+ monkeypatch.setattr(
+ inst,
+ "_start_deployment",
+ lambda manifest: (_ for _ in ()).throw(click.ClickException("startup failed")),
+ )
+ monkeypatch.setattr(inst, "_restore_deployment", lambda manifest: calls.append("restore-old"))
+ monkeypatch.setattr(
+ inst, "delete_recovery_manifest", lambda profile: calls.append("delete-snapshot")
+ )
+
+ with pytest.raises(click.ClickException) as exc:
+ inst._apply_manifest(new)
+
+ message = str(exc.value)
+ assert "startup failed" in message
+ assert "new deployment cleanup failed" in message
+ assert "recovery snapshot" in message
+ assert "remove the new owner before restoring the snapshot" in message
+ assert calls == ["snapshot", "remove-old", "save-new", "install-new", "save-new", "remove-new"]
+ assert load_state_manifest("default") is not None
+ recovery = tmp_path / ".headroom" / "deploy" / "default.recovery.json"
+ assert json.loads(recovery.read_text(encoding="utf-8"))["targets"] == ["old"]
+
+
+def test_install_apply_keeps_snapshot_when_active_persistence_fails(
+ monkeypatch, tmp_path: Path
+) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ previous = DeploymentManifest(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=["old"],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ new = DeploymentManifest(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=["new"],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ )
+ calls: list[str] = []
+ monkeypatch.setattr(inst, "load_manifest", lambda profile: previous)
+ monkeypatch.setattr(inst, "_remove_deployment", lambda manifest: calls.append("cleanup"))
+ monkeypatch.setattr(inst, "install_supervisor", lambda manifest, **kwargs: [])
+ saves = 0
+
+ def fail_active_save(manifest):
+ nonlocal saves
+ saves += 1
+ if saves == 1:
+ inst.save_manifest_strict(manifest)
+ return
+ raise OSError("active manifest busy")
+
+ monkeypatch.setattr(inst, "_save_apply_manifest", fail_active_save)
+ monkeypatch.setattr(inst, "_restore_deployment", lambda manifest: calls.append("restore"))
+ monkeypatch.setattr(inst, "delete_recovery_manifest", lambda profile: calls.append("delete"))
+
+ with pytest.raises(click.ClickException) as exc:
+ inst._apply_manifest(new)
+
+ assert "active manifest persistence failed" in str(exc.value)
+ assert "recovery snapshot:" in str(exc.value)
+ assert calls == ["cleanup", "cleanup"]
+ assert load_state_manifest("default") is not None
+ assert (tmp_path / ".headroom" / "deploy" / "default.recovery.json").exists()
+
+
+def test_install_apply_persists_new_manifest_before_supervisor_install(monkeypatch) -> None:
+ new = SimpleNamespace(profile="default", artifacts=[], mutations=[])
+ calls: list[str] = []
+
+ monkeypatch.setattr(inst, "load_manifest", lambda profile: None)
+ monkeypatch.setattr(inst, "_save_apply_manifest", lambda manifest: calls.append("save"))
+ monkeypatch.setattr(
+ inst,
+ "install_supervisor",
+ lambda manifest, **kwargs: (_ for _ in ()).throw(RuntimeError("install failed")),
+ )
+ monkeypatch.setattr(inst, "_remove_deployment", lambda manifest: calls.append("cleanup"))
+
+ with pytest.raises(click.ClickException, match="install failed"):
+ inst._apply_manifest(new)
+
+ assert calls == ["save", "cleanup"]
+
+
+def test_install_apply_without_previous_has_no_recovery_guidance(monkeypatch) -> None:
+ new = SimpleNamespace(profile="default", artifacts=[], mutations=[])
+ monkeypatch.setattr(inst, "load_manifest", lambda profile: None)
+ monkeypatch.setattr(inst, "_save_apply_manifest", lambda manifest: None)
+ monkeypatch.setattr(inst, "install_supervisor", lambda manifest, **kwargs: [])
+ monkeypatch.setattr(
+ inst,
+ "_start_deployment",
+ lambda manifest: (_ for _ in ()).throw(click.ClickException("startup failed")),
+ )
+ monkeypatch.setattr(
+ inst,
+ "_remove_deployment",
+ lambda manifest: (_ for _ in ()).throw(RuntimeError("cleanup failed")),
+ )
+
+ with pytest.raises(click.ClickException) as exc:
+ inst._apply_manifest(new)
+
+ message = str(exc.value)
+ assert "startup failed" in message and "cleanup failed" in message
+ assert "recovery snapshot" not in message
+
+
def test_install_start_rejects_task_lifecycle(monkeypatch) -> None:
runner = CliRunner()
@@ -995,7 +1519,7 @@ def test_install_apply_uses_docker_runtime_for_persistent_docker(monkeypatch) ->
monkeypatch.setattr("headroom.cli.install.build_manifest", lambda **_: Manifest())
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda deployment: "stopped")
@@ -1012,7 +1536,7 @@ def test_install_apply_uses_docker_runtime_for_persistent_docker(monkeypatch) ->
lambda deployment: calls.append("start_docker"),
)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda deployment: "stopped")
@@ -1065,7 +1589,7 @@ def test_deploy_prefers_docker_when_available(monkeypatch) -> None:
monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build)
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda deployment: "stopped")
@@ -1082,7 +1606,7 @@ def test_deploy_prefers_docker_when_available(monkeypatch) -> None:
lambda deployment: calls.append("start_docker"),
)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
result = runner.invoke(main, ["deploy"])
@@ -1129,7 +1653,7 @@ def test_deploy_prefers_gpu_docker_when_available(monkeypatch) -> None:
monkeypatch.setattr("headroom.cli.install.build_manifest", fake_build)
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: None)
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
- monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment: [])
+ monkeypatch.setattr("headroom.cli.install.install_supervisor", lambda deployment, **kwargs: [])
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda deployment: "stopped")
@@ -1143,7 +1667,7 @@ def test_deploy_prefers_gpu_docker_when_available(monkeypatch) -> None:
monkeypatch.setattr("headroom.cli.install.acquire_runtime_start_lock", fake_lock)
monkeypatch.setattr("headroom.cli.install.start_persistent_docker", lambda deployment: None)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
result = runner.invoke(main, ["deploy"])
@@ -1181,7 +1705,7 @@ def test_deploy_falls_back_to_detached_python_without_supervisor(monkeypatch) ->
monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda deployment: [])
monkeypatch.setattr(
"headroom.cli.install.install_supervisor",
- lambda deployment: calls.append(f"supervisor:{deployment.supervisor_kind}") or [],
+ lambda deployment, **kwargs: calls.append(f"supervisor:{deployment.supervisor_kind}") or [],
)
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda deployment: None)
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
@@ -1199,7 +1723,7 @@ def test_deploy_falls_back_to_detached_python_without_supervisor(monkeypatch) ->
lambda profile: calls.append(f"agent:{profile}"),
)
monkeypatch.setattr(
- "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45: True
+ "headroom.cli.install.wait_ready", lambda deployment, timeout_seconds=45, **kwargs: True
)
result = runner.invoke(main, ["deploy", "--no-docker"])
@@ -1210,7 +1734,7 @@ def test_deploy_falls_back_to_detached_python_without_supervisor(monkeypatch) ->
assert calls == ["supervisor:none", "agent:default"]
-def test_install_remove_continues_when_runtime_teardown_errors(monkeypatch) -> None:
+def test_install_remove_retains_manifest_when_runtime_teardown_errors(monkeypatch) -> None:
runner = CliRunner()
calls: list[str] = []
@@ -1248,8 +1772,10 @@ def test_install_remove_continues_when_runtime_teardown_errors(monkeypatch) -> N
result = runner.invoke(main, ["install", "remove"])
- assert result.exit_code == 0, result.output
- assert calls == ["revert", "remove_supervisor", "delete"]
+ assert result.exit_code != 0
+ assert "Error: Failed to remove deployment 'default': cleanup failed:" in result.output
+ assert "Traceback" not in result.output
+ assert calls == ["revert", "remove_supervisor"]
def test_install_agent_ensure_reports_already_healthy(monkeypatch) -> None:
@@ -1262,8 +1788,10 @@ def test_install_agent_ensure_reports_already_healthy(monkeypatch) -> None:
tool_envs = {}
profile = "default"
health_url = "http://127.0.0.1:8787/readyz"
+ mutations = []
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())
+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: True)
result = runner.invoke(main, ["install", "agent", "ensure"])
@@ -1305,6 +1833,7 @@ def test_install_agent_ensure_no_spawn_when_lock_not_acquired(monkeypatch) -> No
health_url = "http://127.0.0.1:8787/readyz"
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())
+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
import contextlib
@@ -1355,7 +1884,13 @@ def test_install_agent_ensure_stops_wedged_runtime_before_restart(monkeypatch) -
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")
- monkeypatch.setattr("headroom.cli.install.wait_ready", lambda manifest, timeout_seconds: False)
+ wait_calls: list[dict[str, object]] = []
+
+ def fake_wait_ready(manifest, timeout_seconds, **kwargs):
+ wait_calls.append(kwargs)
+ return False
+
+ monkeypatch.setattr("headroom.cli.install.wait_ready", fake_wait_ready)
monkeypatch.setattr(
"headroom.cli.install.revert_mutations", lambda manifest: calls.append("revert")
)
@@ -1391,6 +1926,7 @@ def test_install_agent_ensure_stops_wedged_runtime_before_restart(monkeypatch) -
assert calls.index("revert") < calls.index("stop")
assert calls.index("stop") < calls.index("start_deployment")
assert calls.index("start_deployment") < calls.index("apply")
+ assert wait_calls == [{"require_identity": True}]
assert "start_agent" not in calls
assert "start_docker" not in calls
@@ -1435,7 +1971,9 @@ def test_install_agent_ensure_starts_when_stopped_and_lock_acquired(monkeypatch)
yield True
monkeypatch.setattr("headroom.cli.install.acquire_runtime_start_lock", fake_lock)
- monkeypatch.setattr("headroom.cli.install.wait_ready", lambda manifest, timeout_seconds: True)
+ monkeypatch.setattr(
+ "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds, **kwargs: True
+ )
result = runner.invoke(main, ["install", "agent", "ensure"])
assert result.exit_code == 0, result.output
@@ -1458,6 +1996,7 @@ def test_install_agent_ensure_no_duplicate_spawn_after_lock_recheck(monkeypatch)
# First probe_ready (before lock) returns False, second (after lock) returns True
probe_results = iter([False, True])
monkeypatch.setattr("headroom.cli.install.load_manifest", lambda profile: Manifest())
+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "running")
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: next(probe_results))
monkeypatch.setattr(
@@ -1557,7 +2096,9 @@ def test_install_start_reconciles_a_healthy_deployment(monkeypatch) -> None:
)
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda m: None)
monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda m: None)
- monkeypatch.setattr("headroom.cli.install.wait_ready", lambda m, timeout_seconds=None: True)
+ monkeypatch.setattr(
+ "headroom.cli.install.wait_ready", lambda m, timeout_seconds=None, **kwargs: True
+ )
result = runner.invoke(main, ["install", "start"])
@@ -1597,7 +2138,9 @@ def test_install_start_still_skips_when_nothing_is_pending(monkeypatch) -> None:
)
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda m: None)
monkeypatch.setattr("headroom.cli.install.start_supervisor", lambda m: None)
- monkeypatch.setattr("headroom.cli.install.wait_ready", lambda m, timeout_seconds=None: True)
+ monkeypatch.setattr(
+ "headroom.cli.install.wait_ready", lambda m, timeout_seconds=None, **kwargs: True
+ )
result = runner.invoke(main, ["install", "start"])
diff --git a/tests/test_cli/test_wrap_persistent.py b/tests/test_cli/test_wrap_persistent.py
index 92472881e..6f156308f 100644
--- a/tests/test_cli/test_wrap_persistent.py
+++ b/tests/test_cli/test_wrap_persistent.py
@@ -318,6 +318,28 @@ def test_recover_persistent_proxy_warns_for_task_deployment(monkeypatch) -> None
assert wrap_cli._recover_persistent_proxy(8787) is False
+def test_recover_persistent_proxy_routes_docker_task_to_docker_supervisor(monkeypatch) -> None:
+ calls: list[str] = []
+
+ class DockerTaskManifest(_Manifest):
+ preset = "persistent-task"
+ runtime_kind = "docker"
+
+ monkeypatch.setattr(wrap_cli, "_find_persistent_manifest", lambda port: DockerTaskManifest())
+ monkeypatch.setattr("headroom.install.health.probe_ready", lambda url: False)
+ monkeypatch.setattr("headroom.install.runtime.runtime_ready", lambda manifest: False)
+ monkeypatch.setattr(
+ "headroom.install.runtime.start_persistent_docker",
+ lambda manifest: calls.append("docker"),
+ )
+ monkeypatch.setattr(
+ "headroom.install.runtime.wait_ready", lambda manifest, timeout_seconds=45: True
+ )
+
+ assert wrap_cli._recover_persistent_proxy(8787) is True
+ assert calls == ["docker"]
+
+
def test_ensure_proxy_restarts_idle_stale_ephemeral_proxy(monkeypatch) -> None:
calls: list[object] = []
health = {
diff --git a/tests/test_install/test_paths.py b/tests/test_install/test_paths.py
index 2ddefbdd8..8fa6be370 100644
--- a/tests/test_install/test_paths.py
+++ b/tests/test_install/test_paths.py
@@ -22,6 +22,9 @@ def test_profile_and_artifact_paths(monkeypatch, tmp_path: Path) -> None:
assert install_paths.deploy_root() == tmp_path / "deploy"
assert install_paths.profile_root("demo") == tmp_path / "deploy" / "demo"
assert install_paths.manifest_path("demo") == tmp_path / "deploy" / "demo" / "manifest.json"
+ assert install_paths.recovery_manifest_path("demo") == (
+ tmp_path / "deploy" / "demo.recovery.json"
+ )
assert install_paths.log_path("demo") == tmp_path / "deploy" / "demo" / "runner.log"
assert install_paths.pid_path("demo") == tmp_path / "deploy" / "demo" / "runner.pid"
assert (
diff --git a/tests/test_install/test_planner.py b/tests/test_install/test_planner.py
index d5169acee..dfe5f4525 100644
--- a/tests/test_install/test_planner.py
+++ b/tests/test_install/test_planner.py
@@ -1,5 +1,7 @@
from __future__ import annotations
+import sys
+
import click
import pytest
@@ -54,6 +56,33 @@ def test_build_manifest_for_persistent_docker_sets_expected_defaults() -> None:
assert "HEADROOM_KOMPRESS_BACKEND" not in manifest.base_env
+def test_build_manifest_docker_runtime_never_installs_native_supervisor() -> None:
+ manifest = build_manifest(
+ profile="default",
+ preset=InstallPreset.PERSISTENT_SERVICE.value,
+ runtime_kind="docker",
+ scope="user",
+ provider_mode="manual",
+ targets=[],
+ port=8787,
+ backend="anthropic",
+ anyllm_provider=None,
+ region=None,
+ proxy_mode="token",
+ memory_enabled=False,
+ telemetry_enabled=False,
+ image="ghcr.io/headroomlabs-ai/headroom:latest",
+ )
+
+ assert manifest.supervisor_kind == "none"
+ expected_preset = (
+ InstallPreset.PERSISTENT_TASK.value
+ if sys.platform.startswith("win")
+ else InstallPreset.PERSISTENT_SERVICE.value
+ )
+ assert manifest.preset == expected_preset
+
+
def test_build_manifest_python_runtime_keeps_explicit_memory_db_path() -> None:
manifest = build_manifest(
profile="default",
diff --git a/tests/test_install/test_providers.py b/tests/test_install/test_providers.py
index 837847150..ccbf30dcd 100644
--- a/tests/test_install/test_providers.py
+++ b/tests/test_install/test_providers.py
@@ -432,6 +432,65 @@ def test_apply_mutations_runs_openclaw_for_user_scope(monkeypatch, tmp_path: Pat
assert [mutation.kind for mutation in mutations] == ["openclaw-wrap"]
+def test_provider_mutation_failure_reverts_prior_returned_record(
+ monkeypatch, tmp_path: Path
+) -> None:
+ import headroom.providers.install_registry as registry
+
+ manifest = _manifest(tmp_path)
+ manifest.targets = ["first", "second"]
+ first = ManagedMutation(target="first", kind="first")
+ reverted: list[ManagedMutation] = []
+
+ def apply_first(current):
+ return first
+
+ def fail_second(current):
+ raise RuntimeError("second mutation failed")
+
+ def revert(mutation, current):
+ reverted.append(mutation)
+
+ monkeypatch.setattr(
+ registry,
+ "_PROVIDER_SCOPE_HANDLERS",
+ {"first": (apply_first, revert), "second": (fail_second, revert)},
+ )
+
+ with pytest.raises(RuntimeError, match="second mutation failed"):
+ registry.apply_provider_scope_mutations(manifest)
+ assert reverted == [first]
+
+
+def test_provider_rollback_failure_preserves_unreverted_records(
+ monkeypatch, tmp_path: Path
+) -> None:
+ import headroom.providers.install_registry as registry
+
+ manifest = _manifest(tmp_path)
+ manifest.targets = ["first", "second"]
+ first = ManagedMutation(target="first", kind="first")
+
+ def apply_first(current):
+ return first
+
+ def fail_second(current):
+ raise RuntimeError("second mutation failed")
+
+ def fail_revert(mutation, current):
+ raise RuntimeError("rollback unavailable")
+
+ monkeypatch.setattr(
+ registry,
+ "_PROVIDER_SCOPE_HANDLERS",
+ {"first": (apply_first, fail_revert), "second": (fail_second, fail_revert)},
+ )
+
+ with pytest.raises(RuntimeError, match="rollback unavailable"):
+ registry.apply_provider_scope_mutations(manifest)
+ assert manifest.mutations == [first]
+
+
def test_claude_build_install_env_returns_proxy_base_url() -> None:
# Arrange / Act
env = build_claude_install_env(port=5566, backend="ignored")
@@ -937,3 +996,76 @@ def test_revert_opencode_provider_scope_fallback_on_oserror(monkeypatch, tmp_pat
assert backup_path.exists() # backup preserved when copy fails
assert not config_path.exists() or "headroom" not in config_path.read_text()
+
+
+def test_apply_mutations_records_each_provider_mutation_once(monkeypatch, tmp_path: Path) -> None:
+ import headroom.providers.install_registry as registry
+ from headroom.install.providers import apply_mutations
+
+ manifest = _manifest(tmp_path)
+ manifest.targets = ["first"]
+ first = ManagedMutation(target="first", kind="first")
+ monkeypatch.setattr(
+ registry,
+ "_PROVIDER_SCOPE_HANDLERS",
+ {"first": (lambda current: first, lambda mutation, current: None)},
+ )
+
+ assert apply_mutations(manifest) == [first]
+ assert manifest.mutations == [first]
+
+
+def test_unix_env_failure_restores_previous_shell_contents(monkeypatch, tmp_path: Path) -> None:
+ from headroom import fsutil
+ from headroom.install import providers
+
+ manifest = _manifest(tmp_path)
+ manifest.scope = "user"
+ bashrc = tmp_path / ".bashrc"
+ zshrc = tmp_path / ".zshrc"
+ previous = (
+ '# >>> headroom persistent env >>>\nexport OLD="1"\n# <<< headroom persistent env <<<\n'
+ )
+ bashrc.write_text(previous, encoding="utf-8")
+ monkeypatch.setattr(providers, "unix_user_env_targets", lambda: [bashrc, zshrc])
+ real_write = fsutil.write_text
+
+ def write_text(path, content):
+ if Path(path) == zshrc:
+ raise OSError("disk full")
+ real_write(path, content)
+
+ monkeypatch.setattr(fsutil, "write_text", write_text)
+
+ with pytest.raises(OSError, match="disk full"):
+ providers._apply_unix_env_scope(manifest)
+ assert bashrc.read_text(encoding="utf-8") == previous
+ assert not zshrc.exists()
+
+
+def test_unix_env_failure_restores_earlier_targets_when_existing_target_rejects_writes(
+ monkeypatch, tmp_path: Path
+) -> None:
+ from headroom import fsutil
+ from headroom.install import providers
+
+ manifest = _manifest(tmp_path)
+ manifest.scope = "user"
+ bashrc = tmp_path / ".bashrc"
+ zshrc = tmp_path / ".zshrc"
+ bashrc.write_text("# bash\n", encoding="utf-8")
+ zshrc.write_text("# zsh\n", encoding="utf-8")
+ monkeypatch.setattr(providers, "unix_user_env_targets", lambda: [bashrc, zshrc])
+ real_write = fsutil.write_text
+
+ def write_text(path, content):
+ if Path(path) == zshrc:
+ raise OSError("read-only")
+ real_write(path, content)
+
+ monkeypatch.setattr(fsutil, "write_text", write_text)
+
+ with pytest.raises(OSError, match="read-only"):
+ providers._apply_unix_env_scope(manifest)
+ assert bashrc.read_text(encoding="utf-8") == "# bash\n"
+ assert zshrc.read_text(encoding="utf-8") == "# zsh\n"
diff --git a/tests/test_install/test_runtime.py b/tests/test_install/test_runtime.py
index 046b6a090..183e7a21c 100644
--- a/tests/test_install/test_runtime.py
+++ b/tests/test_install/test_runtime.py
@@ -14,6 +14,8 @@ from headroom.install.runtime import (
_clear_pid,
_deployment_env,
_mount_source,
+ _process_identity,
+ _process_matches_runtime,
_read_pid,
_runtime_env,
_write_pid,
@@ -21,6 +23,7 @@ from headroom.install.runtime import (
build_runtime_command,
resolve_headroom_command,
run_foreground,
+ runtime_ownership,
runtime_status,
start_detached_agent,
start_persistent_docker,
@@ -187,7 +190,7 @@ def test_build_runtime_command_preserves_non_ascii_ambient_token(
captured: dict[str, list[str] | dict[str, str]] = {}
- def fake_run(command: list[str], **kwargs: object) -> None:
+ def fake_run(command: list[str], **kwargs: object) -> subprocess.CompletedProcess:
captured["command"] = command
container_env: dict[str, str] = {}
for index, arg in enumerate(command):
@@ -200,6 +203,7 @@ def test_build_runtime_command_preserves_non_ascii_ambient_token(
else:
container_env[value] = os.environ[value]
captured["container_env"] = container_env
+ return subprocess.CompletedProcess(command, 0)
monkeypatch.setattr("headroom.install.runtime.subprocess.run", fake_run)
start_persistent_docker(manifest)
@@ -548,8 +552,9 @@ def test_runtime_start_lock_blocks_another_process(monkeypatch, tmp_path: Path)
assert result.stdout.strip() == "False"
-def test_run_foreground_and_detached_helpers(monkeypatch, tmp_path: Path) -> None:
+def test_non_launchd_foreground_and_detached_helpers(monkeypatch, tmp_path: Path) -> None:
monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "win32")
monkeypatch.setattr(
"headroom.install.runtime.build_runtime_command", lambda manifest: ["headroom", "proxy"]
)
@@ -631,6 +636,410 @@ def test_run_foreground_and_detached_helpers(monkeypatch, tmp_path: Path) -> Non
assert start_detached_agent("demo") is fake_proc_posix
+def test_launchd_exec_handoff_uses_current_pid_and_never_popens(
+ monkeypatch, tmp_path: Path
+) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")
+ monkeypatch.setattr(
+ "headroom.install.runtime.build_runtime_command",
+ lambda manifest: ["/usr/bin/python3", "-m", "headroom.cli", "proxy"],
+ )
+ expected_env = {"HEADROOM_TEST": "exec"}
+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: expected_env)
+ monkeypatch.setattr("headroom.install.runtime.os.getpid", lambda: 4321)
+
+ exec_calls: list[tuple[str, list[str], dict[str, str]]] = []
+ pid_at_exec: list[int | None] = []
+
+ def fake_execvpe(file: str, args: list[str], env: dict[str, str]) -> None:
+ exec_calls.append((file, args, env))
+ pid_at_exec.append(_read_pid("default"))
+ raise OSError("exec failed")
+
+ monkeypatch.setattr("headroom.install.runtime.os.execvpe", fake_execvpe)
+ monkeypatch.setattr("headroom.install.runtime.os.dup2", lambda source, target: None)
+ monkeypatch.setattr(
+ "headroom.install.runtime.subprocess.Popen",
+ lambda *args, **kwargs: pytest.fail("launchd handoff must not spawn a child"),
+ )
+
+ with pytest.raises(OSError, match="exec failed"):
+ run_foreground(_python_service_manifest())
+
+ assert exec_calls == [
+ (
+ "/usr/bin/python3",
+ ["/usr/bin/python3", "-m", "headroom.cli", "proxy"],
+ expected_env,
+ )
+ ]
+ assert pid_at_exec == [4321]
+ assert _read_pid("default") is None
+
+
+def test_launchd_exec_preserves_runtime_contract(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")
+ command = ["/usr/bin/python3", "-m", "headroom.cli", "proxy", "--port", "8787"]
+ env = {"HEADROOM_DEPLOYMENT_PROFILE": "default", "HEADROOM_SECRET": "test"}
+ monkeypatch.setattr("headroom.install.runtime.build_runtime_command", lambda manifest: command)
+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: env)
+ monkeypatch.setattr("headroom.install.runtime.os.getpid", lambda: 9876)
+
+ dup2_calls: list[tuple[int, int]] = []
+ exec_calls: list[tuple[str, list[str], dict[str, str]]] = []
+
+ class FakeLog:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, exc_type, exc_value, traceback) -> None:
+ return None
+
+ def fileno(self) -> int:
+ return 42
+
+ monkeypatch.setattr(
+ "headroom.install.runtime.open", lambda *args, **kwargs: FakeLog(), raising=False
+ )
+
+ def fake_dup2(source: int, target: int) -> None:
+ dup2_calls.append((source, target))
+
+ def fake_execvpe(file: str, args: list[str], runtime_env: dict[str, str]) -> None:
+ exec_calls.append((file, args, runtime_env))
+ raise OSError("exec failed")
+
+ monkeypatch.setattr("headroom.install.runtime.os.dup2", fake_dup2)
+ monkeypatch.setattr("headroom.install.runtime.os.execvpe", fake_execvpe)
+ monkeypatch.setattr(
+ "headroom.install.runtime.subprocess.Popen",
+ lambda *args, **kwargs: pytest.fail("launchd handoff must not spawn a child"),
+ )
+
+ with pytest.raises(OSError, match="exec failed"):
+ run_foreground(_python_service_manifest())
+
+ assert exec_calls == [(command[0], command, env)]
+ assert len(dup2_calls) == 2
+ assert dup2_calls[0][1:] == (1,)
+ assert dup2_calls[1][1:] == (2,)
+ assert dup2_calls == [(42, 1), (42, 2)]
+
+
+def test_docker_runtime_rejects_run_foreground_split_owner(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")
+ command = ["docker", "run", "--rm", "headroom:test"]
+ env = {"HEADROOM_DEPLOYMENT_RUNTIME": "docker"}
+ monkeypatch.setattr("headroom.install.runtime.build_runtime_command", lambda manifest: command)
+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: env)
+
+ monkeypatch.setattr(
+ "headroom.install.runtime.subprocess.Popen",
+ lambda *args, **kwargs: pytest.fail("Docker supervisor must own the container lifecycle"),
+ )
+ monkeypatch.setattr(
+ "headroom.install.runtime.os.execvpe",
+ lambda *args: pytest.fail("Docker launchd service must keep the daemon-owned Popen path"),
+ )
+
+ manifest = _python_service_manifest()
+ manifest.runtime_kind = "docker"
+ with pytest.raises(RuntimeError, match="Docker deployments must be started"):
+ run_foreground(manifest)
+
+
+def test_runtime_ownership_classifier_covers_launchd_docker_and_foreground(monkeypatch) -> None:
+ python_manifest = _python_service_manifest()
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")
+ assert runtime_ownership(python_manifest) == "launchd-exec"
+
+ for preset, supervisor in (
+ (InstallPreset.PERSISTENT_DOCKER.value, "none"),
+ (InstallPreset.PERSISTENT_SERVICE.value, "service"),
+ (InstallPreset.PERSISTENT_TASK.value, "task"),
+ ):
+ docker_manifest = _python_service_manifest()
+ docker_manifest.preset = preset
+ docker_manifest.runtime_kind = "docker"
+ docker_manifest.supervisor_kind = supervisor
+ assert runtime_ownership(docker_manifest) == "docker-supervisor"
+
+ python_manifest.preset = "persistent-task"
+ python_manifest.supervisor_kind = "task"
+ assert runtime_ownership(python_manifest) == "popen"
+
+
+def test_launchd_transition_keeps_one_listener_owner(monkeypatch, tmp_path: Path) -> None:
+ """A launchd restart replaces the listener in place instead of orphaning a child."""
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")
+ monkeypatch.setattr(
+ "headroom.install.runtime.build_runtime_command",
+ lambda manifest: ["/usr/bin/python3", "-m", "headroom.cli", "proxy"],
+ )
+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: {})
+ monkeypatch.setattr("headroom.install.runtime.os.getpid", lambda: 4321)
+ monkeypatch.setattr("headroom.install.runtime.os.dup2", lambda source, target: None)
+ monkeypatch.setattr("headroom.install.runtime.subprocess.Popen", lambda *a, **k: pytest.fail())
+
+ exec_pids: list[int | None] = []
+
+ class ExecHandoff(Exception):
+ pass
+
+ monkeypatch.setattr(
+ "headroom.install.runtime.os.execvpe",
+ lambda file, args, env: (exec_pids.append(_read_pid("default")), raise_exec()),
+ )
+ monkeypatch.setattr("headroom.install.runtime._clear_pid", lambda *args, **kwargs: None)
+
+ def raise_exec() -> None:
+ raise ExecHandoff
+
+ manifest = _python_service_manifest()
+ assert runtime_ownership(manifest) == "launchd-exec"
+ with pytest.raises(ExecHandoff):
+ run_foreground(manifest)
+ with pytest.raises(ExecHandoff):
+ run_foreground(manifest)
+
+ assert exec_pids == [4321, 4321]
+ assert _read_pid("default") == 4321
+
+
+def test_status_and_stop_reject_a_reused_pid_without_signaling(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _python_service_manifest()
+ _write_pid("default", 123)
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+ monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: False)
+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, current: False)
+ monkeypatch.setattr(
+ "headroom.install.runtime.os.kill",
+ lambda *args: pytest.fail("reused PID must not be signaled"),
+ )
+ assert runtime_status(manifest) == "stopped"
+ _write_pid("default", 123)
+ stop_runtime(manifest)
+ assert _read_pid("default") is None
+
+
+def test_process_identity_must_match_headroom_command(monkeypatch) -> None:
+ class FakeProcess:
+ def __init__(self, pid: int) -> None:
+ self.pid = pid
+
+ def cmdline(self) -> list[str]:
+ return ["python", "unrelated-service.py", "--port", "8787"]
+
+ def environ(self) -> dict[str, str]:
+ return {
+ "HEADROOM_DEPLOYMENT_PROFILE": "default",
+ "HEADROOM_DEPLOYMENT_RUNTIME": "python",
+ }
+
+ monkeypatch.setitem(sys.modules, "psutil", types.SimpleNamespace(Process=FakeProcess))
+ assert not _process_matches_runtime(4321, _python_service_manifest())
+
+ class MatchingProcess(FakeProcess):
+ def cmdline(self) -> list[str]:
+ return ["python", "-m", "headroom.cli", "proxy", "--port", "8787"]
+
+ def environ(self) -> dict[str, str]:
+ return {
+ "HEADROOM_DEPLOYMENT_PROFILE": "default",
+ "HEADROOM_DEPLOYMENT_RUNTIME": "python",
+ }
+
+ monkeypatch.setitem(sys.modules, "psutil", types.SimpleNamespace(Process=MatchingProcess))
+ assert _process_matches_runtime(4321, _python_service_manifest())
+
+
+def test_process_identity_falls_back_without_psutil(monkeypatch) -> None:
+ manifest = _python_service_manifest()
+ monkeypatch.setattr(
+ "headroom.install.runtime._read_proc_metadata",
+ lambda pid: (
+ ["python", "-m", "headroom.cli", "proxy", "--port", "8787"],
+ {
+ "HEADROOM_DEPLOYMENT_PROFILE": "default",
+ "HEADROOM_DEPLOYMENT_RUNTIME": "python",
+ },
+ ),
+ )
+ monkeypatch.setitem(sys.modules, "psutil", None)
+ assert _process_matches_runtime(4321, manifest)
+
+
+def test_process_identity_uses_macos_ps_fallback_without_psutil(monkeypatch) -> None:
+ manifest = _python_service_manifest()
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")
+ monkeypatch.setitem(sys.modules, "psutil", None)
+
+ class Result:
+ returncode = 0
+ stdout = (
+ "/usr/bin/python3 -m headroom.cli proxy --port 8787 "
+ "HEADROOM_DEPLOYMENT_PROFILE=default "
+ "HEADROOM_DEPLOYMENT_RUNTIME=python"
+ )
+
+ monkeypatch.setattr("headroom.install.runtime.run", lambda *a, **k: Result())
+ assert _process_matches_runtime(4321, manifest)
+
+
+def test_process_identity_uses_windows_wmi_command_line_without_psutil(monkeypatch) -> None:
+ manifest = _python_service_manifest()
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "win32")
+ monkeypatch.setitem(sys.modules, "psutil", None)
+
+ class Result:
+ returncode = 0
+ stdout = (
+ '"C:\\Python\\python.exe" -m headroom.cli proxy --port 8787 '
+ "--headroom-deployment-profile default --headroom-deployment-runtime python"
+ )
+
+ calls: list[list[str]] = []
+ monkeypatch.setattr(
+ "headroom.install.runtime.run",
+ lambda command, **kwargs: calls.append(command) or Result(),
+ )
+
+ assert _process_matches_runtime(4321, manifest)
+ assert calls[0][:3] == ["powershell.exe", "-NoProfile", "-NonInteractive"]
+
+
+def test_windows_runtime_status_and_stop_use_wmi_identity(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "win32")
+ monkeypatch.setitem(sys.modules, "psutil", None)
+ manifest = _python_service_manifest()
+ _write_pid("default", 4321)
+ pid_states = iter([True, True, False, False])
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: next(pid_states))
+
+ class Result:
+ returncode = 0
+ stdout = (
+ '"C:\\Python\\python.exe" -m headroom.cli proxy --port 8787 '
+ "--headroom-deployment-profile default --headroom-deployment-runtime python"
+ )
+
+ monkeypatch.setattr("headroom.install.runtime.run", lambda *a, **k: Result())
+ killed: list[tuple[int, int]] = []
+ monkeypatch.setattr(
+ "headroom.install.runtime.os.kill", lambda pid, sig: killed.append((pid, sig))
+ )
+
+ assert runtime_status(manifest) == "running"
+ stop_runtime(manifest)
+ assert killed == [(4321, signal.SIGTERM)]
+ assert _read_pid("default") is None
+
+
+def test_windows_wmi_same_port_without_deployment_markers_is_unknown_and_safe(
+ monkeypatch, tmp_path: Path
+) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "win32")
+ monkeypatch.setitem(sys.modules, "psutil", None)
+ manifest = _python_service_manifest()
+
+ class Result:
+ returncode = 0
+ stdout = '"C:\\Python\\python.exe" -m headroom.cli proxy --port 8787'
+
+ monkeypatch.setattr("headroom.install.runtime.run", lambda *a, **k: Result())
+ _write_pid("default", 4321)
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+ monkeypatch.setattr(
+ "headroom.install.runtime.os.kill",
+ lambda *args: pytest.fail("unidentified same-port process must not be signaled"),
+ )
+
+ assert _process_identity(4321, manifest) is None
+ assert runtime_status(manifest) == "unknown"
+ with pytest.raises(RuntimeError, match="identity unknown"):
+ stop_runtime(manifest)
+ assert _read_pid("default") == 4321
+
+
+def test_process_identity_without_supported_metadata_is_unknown_and_safe(
+ monkeypatch, tmp_path: Path
+) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setitem(sys.modules, "psutil", None)
+ monkeypatch.setattr("headroom.install.runtime._read_proc_metadata", lambda pid: None)
+ manifest = _python_service_manifest()
+ _write_pid("default", 4321)
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+ monkeypatch.setattr(
+ "headroom.install.runtime.os.kill",
+ lambda *args: pytest.fail("unknown identity must not be signaled"),
+ )
+
+ assert _process_identity(4321, manifest) is None
+ assert runtime_status(manifest) == "unknown"
+ with pytest.raises(RuntimeError, match="identity unknown"):
+ stop_runtime(manifest)
+ assert _read_pid("default") == 4321
+
+
+def test_runtime_status_does_not_use_ready_probe_for_failed_identity(
+ monkeypatch, tmp_path: Path
+) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _python_service_manifest()
+ _write_pid("default", 4321)
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, m: False)
+ monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: True)
+
+ assert runtime_status(manifest) == "stopped"
+ assert _read_pid("default") is None
+
+
+def test_process_identity_rejects_current_pid(monkeypatch) -> None:
+ monkeypatch.setattr("headroom.install.runtime.os.getpid", lambda: 4321)
+ assert not _process_matches_runtime(4321, _python_service_manifest())
+
+
+def test_darwin_task_preserves_popen_path(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.runtime.sys.platform", "darwin")
+ monkeypatch.setattr(
+ "headroom.install.runtime.build_runtime_command", lambda manifest: ["headroom", "proxy"]
+ )
+ monkeypatch.setattr("headroom.install.runtime._runtime_env", lambda manifest: {"ENV": "1"})
+
+ class FakeProc:
+ pid = 8765
+
+ def wait(self, timeout: int | None = None) -> int:
+ return 0
+
+ popen_calls: list[tuple[list[str], dict]] = []
+ monkeypatch.setattr(
+ "headroom.install.runtime.subprocess.Popen",
+ lambda command, **kwargs: popen_calls.append((command, kwargs)) or FakeProc(),
+ )
+ monkeypatch.setattr(
+ "headroom.install.runtime.os.execvpe",
+ lambda *args: pytest.fail("Darwin task must preserve Popen"),
+ )
+
+ task_manifest = _python_service_manifest()
+ task_manifest.preset = "persistent-task"
+ task_manifest.supervisor_kind = "task"
+ assert run_foreground(task_manifest) == 0
+ assert popen_calls[0][0] == ["headroom", "proxy"]
+ assert _read_pid("default") is None
+
+
def test_start_detached_agent_closes_parent_log_fd(monkeypatch, tmp_path: Path) -> None:
"""The parent must close its copy of the log file after Popen.
@@ -685,7 +1094,10 @@ def test_start_stop_wait_and_runtime_status_branches(monkeypatch, tmp_path: Path
calls: list[list[str]] = []
monkeypatch.setattr(
"headroom.install.runtime.subprocess.run",
- lambda command, **kwargs: calls.append(command) or type("Result", (), {"stdout": ""})(),
+ lambda command, **kwargs: (
+ calls.append(command)
+ or type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()
+ ),
)
monkeypatch.setattr(
"headroom.install.runtime.build_runtime_command",
@@ -749,6 +1161,9 @@ def test_start_stop_wait_and_runtime_status_branches(monkeypatch, tmp_path: Path
monkeypatch.setattr(
"headroom.install.runtime.os.kill", lambda pid, sig: killed.append((pid, sig))
)
+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, manifest: True)
+ pid_states = iter([True, False, False, True, False, False])
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: next(pid_states))
stop_runtime(python_manifest)
assert killed == [(123, signal.SIGTERM)]
assert _read_pid("default") is None
@@ -789,6 +1204,7 @@ def test_start_stop_wait_and_runtime_status_branches(monkeypatch, tmp_path: Path
_write_pid("default", 125)
monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: False)
assert runtime_status(python_manifest) == "stopped"
+ assert _read_pid("default") is None
def test_stop_runtime_for_docker_stops_and_removes_container(monkeypatch) -> None:
@@ -809,7 +1225,10 @@ def test_stop_runtime_for_docker_stops_and_removes_container(monkeypatch) -> Non
monkeypatch.setattr(
"headroom.install.runtime.subprocess.run",
- lambda command, **kwargs: calls.append(command),
+ lambda command, **kwargs: (
+ calls.append(command)
+ or type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()
+ ),
)
stop_runtime(manifest)
@@ -820,6 +1239,101 @@ def test_stop_runtime_for_docker_stops_and_removes_container(monkeypatch) -> Non
]
+def test_stop_runtime_rejects_unknown_identity(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _python_service_manifest()
+ _write_pid("default", 4321)
+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, current: None)
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+
+ with pytest.raises(RuntimeError, match="identity unknown"):
+ stop_runtime(manifest)
+ assert _read_pid("default") == 4321
+
+
+def test_stop_runtime_rejects_malformed_pid_file(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _python_service_manifest()
+ pid_file = tmp_path / ".headroom" / "deploy" / "default" / "runner.pid"
+ pid_file.parent.mkdir(parents=True)
+ pid_file.write_text("not-a-pid", encoding="utf-8")
+
+ with pytest.raises(RuntimeError, match="runner.pid is invalid"):
+ stop_runtime(manifest)
+ assert pid_file.exists()
+
+
+def test_runtime_status_reports_unknown_for_malformed_pid_file(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _python_service_manifest()
+ pid_file = tmp_path / ".headroom" / "deploy" / "default" / "runner.pid"
+ pid_file.parent.mkdir(parents=True)
+ pid_file.write_text("not-a-pid", encoding="utf-8")
+
+ assert runtime_status(manifest) == "unknown"
+ assert pid_file.exists()
+
+
+def test_stop_runtime_rejects_identity_change_before_clearing_pid(
+ monkeypatch, tmp_path: Path
+) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _python_service_manifest()
+ _write_pid("default", 4321)
+ identities = iter([True, False])
+ monkeypatch.setattr(
+ "headroom.install.runtime._process_identity", lambda pid, current: next(identities)
+ )
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+ monkeypatch.setattr("headroom.install.runtime.os.kill", lambda pid, sig: None)
+
+ with pytest.raises(RuntimeError, match="identity changed"):
+ stop_runtime(manifest)
+ assert _read_pid("default") == 4321
+
+
+def test_stop_runtime_keeps_pid_when_sigterm_does_not_stop_process(
+ monkeypatch, tmp_path: Path
+) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _python_service_manifest()
+ _write_pid("default", 4321)
+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, current: True)
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+ monkeypatch.setattr("headroom.install.runtime.os.kill", lambda pid, sig: None)
+ monkeypatch.setattr("headroom.install.runtime.time.sleep", lambda seconds: None)
+
+ with pytest.raises(RuntimeError, match="remained alive"):
+ stop_runtime(manifest)
+ assert _read_pid("default") == 4321
+
+
+def test_stop_runtime_surfaces_docker_stop_failure(monkeypatch) -> None:
+ manifest = DeploymentManifest(
+ profile="default",
+ preset="persistent-docker",
+ runtime_kind="docker",
+ supervisor_kind="none",
+ scope="user",
+ provider_mode="manual",
+ targets=[],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ container_name="headroom-default",
+ )
+
+ class Result:
+ returncode = 1
+ stdout = ""
+ stderr = "permission denied"
+
+ monkeypatch.setattr("headroom.install.runtime.run", lambda *args, **kwargs: Result())
+
+ with pytest.raises(RuntimeError, match="docker stop failed"):
+ stop_runtime(manifest)
+
+
def test_runtime_status_reads_container_and_pid_state(monkeypatch, tmp_path: Path) -> None:
docker_manifest = DeploymentManifest(
profile="default",
@@ -850,6 +1364,7 @@ def test_runtime_status_reads_container_and_pid_state(monkeypatch, tmp_path: Pat
pid_file.parent.mkdir(parents=True)
pid_file.write_text("123", encoding="utf-8")
monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, manifest: True)
python_manifest = DeploymentManifest(
profile="default",
preset="persistent-service",
@@ -892,6 +1407,7 @@ def test_runtime_status_reports_live_pid_without_terminating(monkeypatch, tmp_pa
monkeypatch.setattr("headroom.install.runtime.os.kill", fail_kill)
monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: True)
+ monkeypatch.setattr("headroom.install.runtime._process_identity", lambda pid, manifest: True)
assert runtime_status(_python_service_manifest()) == "running"
assert pid_file.exists() # status left the deployment untouched
@@ -960,6 +1476,17 @@ class TestRestartCurrentDeployment:
assert result["command"] == "headroom install restart --profile default"
assert popen_calls == [] # cannot run docker from inside the container
+ def test_runtime_env_docker_returns_host_command_without_restarting(self, monkeypatch) -> None:
+ from headroom.install import runtime as rt
+
+ monkeypatch.setenv("HEADROOM_DEPLOYMENT_PROFILE", "default")
+ monkeypatch.setenv("HEADROOM_DEPLOYMENT_RUNTIME", "docker")
+ monkeypatch.delenv("HEADROOM_DEPLOYMENT_PRESET", raising=False)
+ monkeypatch.setattr(rt, "load_manifest", lambda profile: None)
+
+ _manifest, mode = rt.detect_current_deployment()
+ assert mode == "docker"
+
def test_task_mode_detected_and_not_restarted(self, monkeypatch) -> None:
"""A persistent-task deployment must not be told it's a self-restartable 'service'.
@@ -1010,3 +1537,17 @@ class TestRestartCurrentDeployment:
assert result["restarted"] is True
assert result["mode"] == "service"
assert recorded["command"][-4:] == ["install", "restart", "--profile", "default"]
+
+
+def test_stop_runtime_clears_pid_of_dead_process(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _python_service_manifest()
+ _write_pid("default", 4321)
+ monkeypatch.setattr("headroom.install.runtime.pid_alive", lambda pid: False)
+ monkeypatch.setattr(
+ "headroom.install.runtime._process_identity",
+ lambda pid, current: pytest.fail("a dead PID has no identity to check"),
+ )
+
+ stop_runtime(manifest)
+ assert _read_pid("default") is None
diff --git a/tests/test_install/test_state.py b/tests/test_install/test_state.py
index 4622e3f04..647cfd37f 100644
--- a/tests/test_install/test_state.py
+++ b/tests/test_install/test_state.py
@@ -9,9 +9,11 @@ from headroom.install.models import ArtifactRecord, DeploymentManifest, ManagedM
from headroom.install.state import (
ManifestError,
delete_manifest,
+ delete_recovery_manifest,
list_manifests,
load_manifest,
save_manifest,
+ save_recovery_manifest,
)
@@ -72,6 +74,22 @@ def test_save_manifest_writes_atomically(monkeypatch, tmp_path: Path) -> None:
assert load_manifest("default") is not None
+def test_recovery_manifest_preserves_mutations_and_artifacts(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ manifest = _manifest()
+
+ save_recovery_manifest(manifest)
+
+ recovery = tmp_path / ".headroom" / "deploy" / "default.recovery.json"
+ assert recovery.exists()
+ payload = json.loads(recovery.read_text(encoding="utf-8"))
+ assert payload["mutations"][0]["kind"] == "shell-block"
+ assert payload["artifacts"][0]["kind"] == "script"
+ assert load_manifest("default") is None
+ delete_recovery_manifest("default")
+ assert not recovery.exists()
+
+
def test_list_manifests_ignores_invalid_payloads(monkeypatch, tmp_path: Path) -> None:
monkeypatch.setattr(Path, "home", lambda: tmp_path)
valid = _manifest()
@@ -154,3 +172,34 @@ def test_delete_manifest_removes_profile_root(monkeypatch, tmp_path: Path) -> No
assert load_manifest("default") is None
assert not extra_file.parent.exists()
+
+
+def test_delete_manifest_propagates_removal_failure(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ root = tmp_path / ".headroom" / "deploy" / "default"
+ root.mkdir(parents=True)
+ monkeypatch.setattr(
+ "headroom.install.state.shutil.rmtree", lambda path: (_ for _ in ()).throw(OSError("busy"))
+ )
+
+ with pytest.raises(OSError, match="busy"):
+ delete_manifest("default")
+
+
+def test_manifest_writes_pin_lf(monkeypatch, tmp_path: Path) -> None:
+ import headroom.install.state as state
+
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ calls: list[dict] = []
+ real_fdopen = state.os.fdopen
+
+ def fdopen(fd, *args, **kwargs):
+ calls.append(kwargs)
+ return real_fdopen(fd, *args, **kwargs)
+
+ monkeypatch.setattr(state.os, "fdopen", fdopen)
+ save_manifest(_manifest())
+ save_recovery_manifest(_manifest())
+
+ assert [call.get("newline") for call in calls] == ["\n", "\n"]
diff --git a/tests/test_install/test_supervisors.py b/tests/test_install/test_supervisors.py
index 400b1a648..f3310bc0f 100644
--- a/tests/test_install/test_supervisors.py
+++ b/tests/test_install/test_supervisors.py
@@ -291,6 +291,8 @@ def test_macos_launchd_plist_switches_between_keepalive_and_interval(
assert service_path == tmp_path / "Library" / "LaunchAgents" / "com.headroom.default.plist"
assert "<key>KeepAlive</key>" in service_content
assert "<key>StartInterval</key>" not in service_content
+ assert "<key>ProgramArguments</key>" in service_content
+ assert "run-headroom.sh" in service_content
task_manifest = _manifest(profile="tasky", supervisor=SupervisorKind.TASK.value)
task_path, task_content = _macos_launchd_plist(
@@ -795,6 +797,32 @@ def test_install_supervisor_darwin_windows_and_unsupported(monkeypatch, tmp_path
install_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
+def test_install_supervisor_darwin_prepare_does_not_bootstrap(monkeypatch, tmp_path: Path) -> None:
+ run_script = tmp_path / "run-headroom.sh"
+ monkeypatch.setattr(
+ "headroom.install.supervisors.render_runner_scripts",
+ lambda manifest: [type("Record", (), {"kind": "script", "path": str(run_script)})()],
+ )
+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")
+ monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 123, raising=False)
+ monkeypatch.setattr(
+ "headroom.install.supervisors._macos_launchd_plist",
+ lambda manifest, script, interval=None: (tmp_path / "job.plist", "plist"),
+ )
+ bootstraps: list[list[str]] = []
+ monkeypatch.setattr(
+ "headroom.install.supervisors._bootstrap_with_retry",
+ lambda domain, path, **kwargs: bootstraps.append([domain, str(path)]),
+ )
+ monkeypatch.setattr(
+ "headroom.install.supervisors.run", lambda *args, **kwargs: _LaunchctlResult(0)
+ )
+
+ install_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value), start=False)
+
+ assert bootstraps == []
+
+
def test_install_supervisor_retries_bootstrap_until_launchd_settles(
monkeypatch, tmp_path: Path
) -> None:
@@ -871,6 +899,16 @@ class _LaunchctlResult:
self.stdout = stdout
+def test_linux_persistent_task_stop_does_not_call_systemctl(monkeypatch) -> None:
+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "linux")
+ monkeypatch.setattr(
+ "headroom.install.supervisors.subprocess.run",
+ lambda *args, **kwargs: pytest.fail("cron task stop must not call systemctl"),
+ )
+
+ stop_supervisor(_manifest(supervisor=SupervisorKind.TASK.value))
+
+
def test_start_and_stop_supervisor_darwin_windows_and_none(monkeypatch) -> None:
calls: list[list[str]] = []
monkeypatch.setattr(
@@ -1041,7 +1079,11 @@ def test_remove_supervisor_linux_service_cron_path_and_missing_crontab(
def fake_run(command: list[str], **kwargs):
calls.append(command)
- return type("Result", (), {"returncode": 1, "stdout": ""})()
+ if command == ["crontab", "-l"]:
+ return type(
+ "Result", (), {"returncode": 1, "stdout": "", "stderr": "no crontab for user"}
+ )()
+ return type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()
monkeypatch.setattr("headroom.install.supervisors.subprocess.run", fake_run)
unit_path = tmp_path / "headroom-default.service"
@@ -1072,11 +1114,35 @@ def test_remove_supervisor_linux_service_cron_path_and_missing_crontab(
assert calls[-1] == ["crontab", "-l"]
+def test_remove_supervisor_surfaces_linux_command_failure(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "linux")
+
+ class Result:
+ returncode = 1
+ stdout = ""
+ stderr = "permission denied"
+
+ monkeypatch.setattr("headroom.install.supervisors.run", lambda *args, **kwargs: Result())
+ unit_path = tmp_path / "headroom-default.service"
+ unit_path.write_text("unit", encoding="utf-8")
+ monkeypatch.setattr(
+ "headroom.install.supervisors._linux_service_unit",
+ lambda manifest, script: (unit_path, "unit"),
+ )
+
+ with pytest.raises(click.ClickException, match="systemctl disable --now failed"):
+ remove_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
+ assert unit_path.exists()
+
+
def test_remove_supervisor_darwin_and_windows(monkeypatch, tmp_path: Path) -> None:
calls: list[list[str]] = []
monkeypatch.setattr(
"headroom.install.supervisors.subprocess.run",
- lambda command, **kwargs: calls.append(command),
+ lambda command, **kwargs: (
+ calls.append(command)
+ or type("Result", (), {"returncode": 0, "stdout": "", "stderr": ""})()
+ ),
)
monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 55, raising=False)