Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(install): repair macOS launchd restart/start lifecycle (#1290)

Grant McNaughtda1a3973ed79
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 08a561473..3ec13e1e7 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -40,7 +40,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
* **proxy:** make `force_kompress` skip ContentRouter auto-detection during compression and pass savings-profile kwargs through Anthropic batch requests.
* **proxy:** add native Bedrock `/model/{id}/converse-stream` route and forward it through the existing streaming EventStream/SSE pipeline.
* **wrap (codex):** fix `headroom wrap codex` producing a `config.toml` with duplicate top-level `model_provider` / `openai_base_url` keys (TOML-spec error) when the user had already configured their own provider. The injector now rewrites pre-existing top-level `model_provider` and `openai_base_url` lines in place — the previous value is kept in a `# was: …` trailing comment — instead of unconditionally prepending a duplicate, so `codex` can start against the proxy. The pre-wrap snapshot mechanism continues to byte-for-byte restore the original file on `headroom unwrap codex`.
-
+* **install (macOS):** fix `headroom install restart` / `install start` for launchd `persistent-service` deployments. `stop` `bootout`s the job but `start` only ran `launchctl kickstart`, which cannot recover the un-bootstrapped state `stop`/`restart` leave behind (launchctl error 113), so the proxy was left stopped. `start` now tries `kickstart` (fast path for an already-bootstrapped job) and, on failure, `bootstrap`s the plist fresh — retrying for ~15s to ride out the transient `bootstrap` EIO (error 5) window while launchd releases the label after a `bootout`. `stop` tolerates only the already-absent case (`bootout` ESRCH / error 3) and still raises on any other `bootout` failure ([#1289](https://github.com/headroomlabs-ai/headroom/issues/1289)).
* **wrap:** isolate wrapped proxy subprocess stdout/stderr into `proxy-stdio.log`, so `proxy.log` remains the canonical rotating runtime log and Windows rollover failures from `RotatingFileHandler` are no longer blocked by wrapper stdio handles ([#1184](https://github.com/chopratejas/headroom/issues/1184)).
* **langchain:** fix `HeadroomChatModel.ainvoke()` crashing with `AttributeError: 'AsyncStream' object has no attribute 'model_dump'` when the wrapped model has `streaming=True`. `_agenerate()` now uses a per-call non-streaming copy of the wrapped model instead of mutating shared state across an `await` ([#1285](https://github.com/headroomlabs-ai/headroom/issues/1285)).
diff --git a/headroom/install/supervisors.py b/headroom/install/supervisors.py
index edd991c8a..b03cba725 100644
--- a/headroom/install/supervisors.py
+++ b/headroom/install/supervisors.py
@@ -7,6 +7,7 @@ import re
import shlex
import subprocess
import sys
+import time
from pathlib import Path
import click
@@ -22,6 +23,16 @@ from .paths import (
)
from .runtime import resolve_headroom_command
+# After `launchctl bootout`, a follow-up `bootstrap` of the same label can
+# return EIO (error 5) for several seconds while launchd releases it. Retry the
+# bootstrap up to ~15s (30 attempts x 0.5s) to ride out that settle window.
+_MACOS_BOOTSTRAP_RETRIES = 30
+_MACOS_BOOTSTRAP_RETRY_DELAY = 0.5
+
+# `launchctl bootout` of an already-absent job exits with ESRCH ("No such
+# process"). That single code is the only failure we treat as already-stopped.
+_LAUNCHCTL_ESRCH = 3
+
def _is_windows() -> bool:
return sys.platform.startswith("win")
@@ -310,8 +321,44 @@ def start_supervisor(manifest: DeploymentManifest) -> None:
and manifest.supervisor_kind == SupervisorKind.SERVICE.value
else f"gui/{os.getuid()}"
)
- subprocess.run(["launchctl", "kickstart", "-k", f"{domain}/{label}"], check=True)
- return
+ # Fast path: when the job is already bootstrapped (e.g. `start` right
+ # after `install apply`, or `start` on a running service), `kickstart`
+ # restarts it in place.
+ kick = subprocess.run(
+ ["launchctl", "kickstart", "-k", f"{domain}/{label}"],
+ capture_output=True,
+ text=True,
+ )
+ if kick.returncode == 0:
+ return
+ # Otherwise the job is not registered in the domain. This is the state
+ # `stop`/`restart` leave behind, since they `bootout` the job, and
+ # `kickstart` cannot recover it (launchctl error 113). Bootstrap fresh
+ # instead — a successful bootstrap also starts the job via RunAtLoad.
+ # launchd can return EIO (error 5) from bootstrap for several seconds
+ # after a bootout while it releases the label, so retry for ~15s.
+ plist_dir = (
+ Path("/Library/LaunchDaemons")
+ if manifest.scope == "system"
+ and manifest.supervisor_kind == SupervisorKind.SERVICE.value
+ else Path.home() / "Library" / "LaunchAgents"
+ )
+ plist_path = plist_dir / f"{label}.plist"
+ last = kick
+ for _ in range(_MACOS_BOOTSTRAP_RETRIES):
+ boot = subprocess.run(
+ ["launchctl", "bootstrap", domain, str(plist_path)],
+ capture_output=True,
+ text=True,
+ )
+ if boot.returncode == 0:
+ return
+ last = boot
+ time.sleep(_MACOS_BOOTSTRAP_RETRY_DELAY)
+ detail = (last.stderr or last.stdout or "").strip()
+ raise click.ClickException(
+ f"launchctl could not start {domain}/{label}: {detail or 'unknown error'}"
+ )
if _is_windows() and manifest.supervisor_kind == SupervisorKind.SERVICE.value:
subprocess.run(["sc.exe", "start", manifest.service_name], check=True)
@@ -333,7 +380,21 @@ def stop_supervisor(manifest: DeploymentManifest) -> None:
and manifest.supervisor_kind == SupervisorKind.SERVICE.value
else f"gui/{os.getuid()}"
)
- subprocess.run(["launchctl", "bootout", f"{domain}/{label}"], check=True)
+ # `bootout` exits with ESRCH ("No such process") when the job is already
+ # absent — tolerate only that, so `restart` can proceed to start again.
+ # Any other non-zero result is a real failure (permissions, malformed
+ # domain, launchd error) and must surface; otherwise `restart` could
+ # report success while a stale job is still running.
+ result = subprocess.run(
+ ["launchctl", "bootout", f"{domain}/{label}"],
+ capture_output=True,
+ text=True,
+ )
+ if result.returncode not in (0, _LAUNCHCTL_ESRCH):
+ detail = (result.stderr or result.stdout or "").strip()
+ raise click.ClickException(
+ f"launchctl bootout failed for {domain}/{label}: {detail or 'unknown error'}"
+ )
return
if _is_windows() and manifest.supervisor_kind == SupervisorKind.SERVICE.value:
subprocess.run(["sc.exe", "stop", manifest.service_name], check=True)
diff --git a/tests/test_install/test_supervisors.py b/tests/test_install/test_supervisors.py
index c77c9b2d6..161cc8629 100644
--- a/tests/test_install/test_supervisors.py
+++ b/tests/test_install/test_supervisors.py
@@ -335,11 +335,18 @@ def test_install_supervisor_darwin_windows_and_unsupported(monkeypatch, tmp_path
install_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
+class _LaunchctlResult:
+ def __init__(self, returncode: int = 0, stderr: str = "", stdout: str = "") -> None:
+ self.returncode = returncode
+ self.stderr = stderr
+ self.stdout = stdout
+
+
def test_start_and_stop_supervisor_darwin_windows_and_none(monkeypatch) -> None:
calls: list[list[str]] = []
monkeypatch.setattr(
"headroom.install.supervisors.subprocess.run",
- lambda command, **kwargs: calls.append(command),
+ lambda command, **kwargs: calls.append(command) or _LaunchctlResult(0),
)
monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 77, raising=False)
@@ -349,6 +356,8 @@ def test_start_and_stop_supervisor_darwin_windows_and_none(monkeypatch) -> None:
monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")
monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")
+ # Warm path: kickstart succeeds (job already bootstrapped), so start does
+ # not fall through to bootstrap.
start_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
stop_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
assert calls == [
@@ -367,6 +376,108 @@ def test_start_and_stop_supervisor_darwin_windows_and_none(monkeypatch) -> None:
]
+def test_macos_start_bootstraps_when_job_not_registered(monkeypatch, tmp_path: Path) -> None:
+ # Post-`stop`/`restart` state: the job was booted out, so `kickstart` fails
+ # (launchctl 113) and start must bootstrap the plist instead.
+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")
+ monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 77, raising=False)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ calls: list[list[str]] = []
+
+ def fake_run(command, **kwargs):
+ calls.append(command)
+ if command[1] == "kickstart":
+ return _LaunchctlResult(113, stderr="Could not find service")
+ return _LaunchctlResult(0)
+
+ monkeypatch.setattr("headroom.install.supervisors.subprocess.run", fake_run)
+
+ start_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
+
+ plist_path = tmp_path / "Library" / "LaunchAgents" / "com.headroom.default.plist"
+ assert calls == [
+ ["launchctl", "kickstart", "-k", "gui/77/com.headroom.default"],
+ ["launchctl", "bootstrap", "gui/77", str(plist_path)],
+ ]
+
+
+def test_macos_start_retries_bootstrap_until_launchd_settles(monkeypatch, tmp_path: Path) -> None:
+ # launchd returns EIO (error 5) from bootstrap for a while after a bootout;
+ # start should retry until it succeeds.
+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")
+ monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 77, raising=False)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.supervisors.time.sleep", lambda _s: None)
+ bootstrap_attempts = 0
+
+ def fake_run(command, **kwargs):
+ nonlocal bootstrap_attempts
+ if command[1] == "kickstart":
+ return _LaunchctlResult(113)
+ bootstrap_attempts += 1
+ if bootstrap_attempts < 3:
+ return _LaunchctlResult(5, stderr="Bootstrap failed: 5: Input/output error")
+ return _LaunchctlResult(0)
+
+ monkeypatch.setattr("headroom.install.supervisors.subprocess.run", fake_run)
+
+ start_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
+ assert bootstrap_attempts == 3
+
+
+def test_macos_start_raises_after_bootstrap_keeps_failing(monkeypatch, tmp_path: Path) -> None:
+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")
+ monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 77, raising=False)
+ monkeypatch.setattr(Path, "home", lambda: tmp_path)
+ monkeypatch.setattr("headroom.install.supervisors.time.sleep", lambda _s: None)
+ monkeypatch.setattr("headroom.install.supervisors._MACOS_BOOTSTRAP_RETRIES", 3)
+
+ def fake_run(command, **kwargs):
+ if command[1] == "kickstart":
+ return _LaunchctlResult(113)
+ return _LaunchctlResult(5, stderr="Bootstrap failed: 5: Input/output error")
+
+ monkeypatch.setattr("headroom.install.supervisors.subprocess.run", fake_run)
+
+ with pytest.raises(click.ClickException, match="could not start"):
+ start_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
+
+
+def test_macos_stop_tolerates_missing_job(monkeypatch) -> None:
+ # `bootout` of an absent job exits with ESRCH (3); stop must not raise so
+ # that `restart` can proceed to start again.
+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")
+ monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 77, raising=False)
+ calls: list[list[str]] = []
+
+ def fake_run(command, **kwargs):
+ calls.append(command)
+ assert kwargs.get("check") is not True
+ return _LaunchctlResult(3, stderr="Boot-out failed: 3: No such process")
+
+ monkeypatch.setattr("headroom.install.supervisors.subprocess.run", fake_run)
+
+ stop_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
+ assert calls == [["launchctl", "bootout", "gui/77/com.headroom.default"]]
+
+
+def test_macos_stop_raises_on_non_esrch_failure(monkeypatch) -> None:
+ # A non-3 `bootout` failure (e.g. permissions) is a real error and must
+ # surface — otherwise `restart` could report success with a stale job still
+ # running.
+ monkeypatch.setattr("headroom.install.supervisors.sys.platform", "darwin")
+ monkeypatch.setattr("headroom.install.supervisors.os.getuid", lambda: 77, raising=False)
+ monkeypatch.setattr(
+ "headroom.install.supervisors.subprocess.run",
+ lambda command, **kwargs: _LaunchctlResult(
+ 9, stderr="Boot-out failed: 9: Operation not permitted"
+ ),
+ )
+
+ with pytest.raises(click.ClickException, match="bootout failed"):
+ stop_supervisor(_manifest(supervisor=SupervisorKind.SERVICE.value))
+
+
def test_remove_supervisor_removes_user_crontab_block(monkeypatch) -> None:
calls: list[tuple[list[str], str | None]] = []
monkeypatch.setattr("headroom.install.supervisors.sys.platform", "linux")