Skip to content

Compress tool outputs, logs, files, and RAG chunks before they reach the LLM. 20% fewer tokens for coding agents, 60-95% fewer tokens for JSON, same answers. Library, proxy, MCP server.

fix(install): wait for the old process to stop before restarting (#3670)

Radhakrishnan Pachyappan11c7320bbcc4
diff --git a/headroom/cli/install.py b/headroom/cli/install.py
index 195e220e4..e8bd77bb8 100644
--- a/headroom/cli/install.py
+++ b/headroom/cli/install.py
@@ -33,6 +33,7 @@ from headroom.install.runtime import (
start_persistent_docker,
stop_runtime,
wait_ready,
+ wait_stopped,
)
from headroom.install.state import (
ManifestError,
@@ -191,6 +192,15 @@ def _stop_deployment(manifest: DeploymentManifest) -> None:
if manifest.supervisor_kind == SupervisorKind.SERVICE.value:
stop_supervisor(manifest)
stop_runtime(manifest)
+ # Stopping returns before the old process has finished shutting down, so it
+ # can keep answering /readyz. `_start_deployment` treats a ready endpoint as
+ # "already running" and would skip the start, leaving the deployment stopped
+ # once the old process exits. Block until it is really gone.
+ if not wait_stopped(manifest):
+ raise click.ClickException(
+ f"Deployment '{manifest.profile}' is still answering on "
+ f"{manifest.health_url} after stop."
+ )
def _deactivate_deployment_mutations(
diff --git a/headroom/install/runtime.py b/headroom/install/runtime.py
index 2abdcd9aa..dd40104da 100644
--- a/headroom/install/runtime.py
+++ b/headroom/install/runtime.py
@@ -395,6 +395,17 @@ def wait_ready(manifest: DeploymentManifest, timeout_seconds: int = 30) -> bool:
return False
+def wait_stopped(manifest: DeploymentManifest, timeout_seconds: int = 15) -> bool:
+ """Wait for the deployment's health endpoint to stop answering."""
+
+ deadline = time.monotonic() + timeout_seconds
+ while probe_ready(manifest.health_url):
+ if time.monotonic() >= deadline:
+ return False
+ time.sleep(0.25)
+ return True
+
+
def runtime_status(manifest: DeploymentManifest) -> str:
"""Return a short status string for the deployment runtime."""
diff --git a/tests/test_cli/test_install_cli.py b/tests/test_cli/test_install_cli.py
index 750fb89ee..b12184bc6 100644
--- a/tests/test_cli/test_install_cli.py
+++ b/tests/test_cli/test_install_cli.py
@@ -543,6 +543,7 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:
)
monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: calls.append("save"))
monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
+ monkeypatch.setattr("headroom.cli.install.wait_stopped", lambda manifest: True)
monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")
result = runner.invoke(main, ["install", "restart"])
@@ -560,6 +561,82 @@ def test_install_restart_uses_internal_helpers(monkeypatch) -> None:
]
+def _restart_race_manifest():
+ class Manifest:
+ profile = "default"
+ preset = "persistent-service"
+ runtime_kind = "python"
+ supervisor_kind = "service"
+ scope = "user"
+ health_url = "http://127.0.0.1:8787/readyz"
+ mutations: list = []
+
+ return Manifest()
+
+
+def test_install_restart_waits_for_old_process_before_starting(monkeypatch) -> None:
+ """#3658: the old proxy still answers /readyz right after stop; restart must
+ wait for it to go away, otherwise start is skipped and the service stays down."""
+
+ runner = CliRunner()
+ calls: list[str] = []
+
+ monkeypatch.setattr(
+ "headroom.cli.install.load_manifest", lambda profile: _restart_race_manifest()
+ )
+ monkeypatch.setattr(
+ "headroom.cli.install.stop_supervisor", lambda manifest: calls.append("stop_supervisor")
+ )
+ monkeypatch.setattr(
+ "headroom.cli.install.stop_runtime", lambda manifest: calls.append("stop_runtime")
+ )
+ monkeypatch.setattr(
+ "headroom.cli.install.wait_stopped",
+ lambda manifest, timeout_seconds=15: calls.append("wait_stopped") or True,
+ )
+ monkeypatch.setattr(
+ "headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
+ )
+ monkeypatch.setattr(
+ "headroom.cli.install.wait_ready", lambda manifest, timeout_seconds=45: True
+ )
+ monkeypatch.setattr("headroom.cli.install.apply_mutations", lambda manifest: [])
+ monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: None)
+ monkeypatch.setattr("headroom.cli.install.probe_ready", lambda url: False)
+ monkeypatch.setattr("headroom.cli.install.runtime_status", lambda manifest: "stopped")
+
+ result = runner.invoke(main, ["install", "restart"])
+
+ assert result.exit_code == 0, result.output
+ assert calls == ["stop_supervisor", "stop_runtime", "wait_stopped", "start_supervisor"]
+
+
[email protected]("command", ["stop", "restart"])
+def test_install_stop_and_restart_fail_when_old_process_never_stops(monkeypatch, command) -> None:
+ runner = CliRunner()
+ calls: list[str] = []
+
+ monkeypatch.setattr(
+ "headroom.cli.install.load_manifest", lambda profile: _restart_race_manifest()
+ )
+ monkeypatch.setattr("headroom.cli.install.stop_supervisor", lambda manifest: None)
+ monkeypatch.setattr("headroom.cli.install.stop_runtime", lambda manifest: None)
+ monkeypatch.setattr(
+ "headroom.cli.install.wait_stopped", lambda manifest, timeout_seconds=15: False
+ )
+ monkeypatch.setattr(
+ "headroom.cli.install.start_supervisor", lambda manifest: calls.append("start_supervisor")
+ )
+ monkeypatch.setattr("headroom.cli.install.save_manifest", lambda manifest: None)
+
+ result = runner.invoke(main, ["install", command])
+
+ assert result.exit_code != 0
+ assert "still answering" in result.output
+ assert "deployment 'default'." not in result.output
+ assert calls == []
+
+
def test_install_start_noops_when_already_healthy(monkeypatch) -> None:
runner = CliRunner()
calls: list[str] = []
diff --git a/tests/test_install/test_runtime.py b/tests/test_install/test_runtime.py
index 4b254cd45..21b60cc48 100644
--- a/tests/test_install/test_runtime.py
+++ b/tests/test_install/test_runtime.py
@@ -26,9 +26,52 @@ from headroom.install.runtime import (
start_persistent_docker,
stop_runtime,
wait_ready,
+ wait_stopped,
)
+def _wait_stopped_manifest() -> DeploymentManifest:
+ return DeploymentManifest(
+ profile="default",
+ preset="persistent-service",
+ runtime_kind="python",
+ supervisor_kind="service",
+ scope="user",
+ provider_mode="manual",
+ targets=[],
+ port=8787,
+ host="127.0.0.1",
+ backend="anthropic",
+ health_url="http://127.0.0.1:8787/readyz",
+ )
+
+
+def test_wait_stopped_waits_for_health_endpoint_to_go_down(monkeypatch) -> None:
+ """#3658: the old process keeps answering /readyz briefly after it is stopped."""
+
+ probe_results = iter([True, True, False])
+ sleeps: list[float] = []
+ monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: next(probe_results))
+ monkeypatch.setattr(
+ "headroom.install.runtime.time.sleep", lambda seconds: sleeps.append(seconds)
+ )
+
+ assert wait_stopped(_wait_stopped_manifest(), timeout_seconds=5) is True
+ assert len(sleeps) == 2
+
+
+def test_wait_stopped_times_out_when_endpoint_keeps_answering(monkeypatch) -> None:
+ clock = {"now": 0.0}
+ monkeypatch.setattr("headroom.install.runtime.probe_ready", lambda url: True)
+ monkeypatch.setattr("headroom.install.runtime.time.monotonic", lambda: clock["now"])
+ monkeypatch.setattr(
+ "headroom.install.runtime.time.sleep",
+ lambda seconds: clock.update(now=clock["now"] + seconds),
+ )
+
+ assert wait_stopped(_wait_stopped_manifest(), timeout_seconds=2) is False
+
+
def test_build_runtime_command_for_docker_includes_deployment_env(
monkeypatch, tmp_path: Path
) -> None: